Deep Learning RNN for Sentiment Classifier.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sentiment Classification Using RNNs \n",
    "\n",
    "### In this example, we see sentiment classification.\n",
    "### Use built-in RNNs: nn.RNN, nn.LSTM, and nn.GRU.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from argparse import Namespace\n",
    "import os\n",
    "import json\n",
    "from collections import Counter\n",
    "import string\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from tqdm.notebook import tqdm\n",
    "import regex as re\n",
    "#added import nltk\n",
    "import nltk\n",
    "from nltk.corpus import stopwords, words\n",
    "from nltk import word_tokenize\n",
    "wnl = nltk.WordNetLemmatizer()\n",
    "correct_words = words.words()\n",
    "import contractions\n",
    "import inflect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pip install contractions inflect"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Vocabulary, Vectorizer, Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Vocabulary(object):\n",
    "    \"\"\"Class to process text and extract vocabulary for mapping\"\"\"\n",
    "\n",
    "    def __init__(self, token_to_idx=None):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            token_to_idx (dict): a pre-existing map of tokens to indices\n",
    "        \"\"\"\n",
    "\n",
    "        if token_to_idx is None:\n",
    "            token_to_idx = {}\n",
    "        self._token_to_idx = token_to_idx\n",
    "\n",
    "        self._idx_to_token = {idx: token \n",
    "                              for token, idx in self._token_to_idx.items()}\n",
    "#         self._idx_to_token = \\\n",
    "#                 {idx: wnl.lemmatize(contractions.fix((re.sub(r'(?!<\\d)\\.(?!\\d)|[^\\s\\w.]',token.lower())))) \\\n",
    "#                 for token, idx in self._token_to_idx.items() if not stopwords.words('english')} \n",
    "        \n",
    "    def add_token(self, token):\n",
    "        \"\"\"Update mapping dicts based on the token.\n",
    "\n",
    "        Args:\n",
    "            token (str): the item to add into the Vocabulary\n",
    "        Returns:\n",
    "            index (int): the integer corresponding to the token\n",
    "        \"\"\"\n",
    "#         token = re.sub(r'(?!<\\d)\\.(?!\\d)|[^\\s\\w.]', '', token.lower())  #added\n",
    "#         if token in stopwords.words('english'):#added\n",
    "#             return -1\n",
    "        if token in self._token_to_idx:\n",
    "            index = self._token_to_idx[token]\n",
    "        else:\n",
    "            index = len(self._token_to_idx)\n",
    "            self._token_to_idx[token] = index\n",
    "            self._idx_to_token[index] = token\n",
    "        return index\n",
    "            \n",
    "    def lookup_token(self, token):\n",
    "        \"\"\"Retrieve the index associated with the token \n",
    "        \n",
    "        Args:\n",
    "            token (str): the token to look up \n",
    "        Returns:\n",
    "            index (int): the index corresponding to the token\n",
    "        \"\"\"\n",
    "        return self._token_to_idx[token]\n",
    "\n",
    "    def lookup_index(self, index):\n",
    "        \"\"\"Return the token associated with the index\n",
    "        \n",
    "        Args: \n",
    "            index (int): the index to look up\n",
    "        Returns:\n",
    "            token (str): the token corresponding to the index\n",
    "        Raises:\n",
    "            KeyError: if the index is not in the Vocabulary\n",
    "        \"\"\"\n",
    "        if index not in self._idx_to_token:\n",
    "            raise KeyError(\"the index (%d) is not in the Vocabulary\" % index)\n",
    "        return self._idx_to_token[index]\n",
    "\n",
    "    def __str__(self):\n",
    "        return \"<Vocabulary(size=%d)>\" % len(self)\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self._token_to_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SequenceVocabulary(Vocabulary):\n",
    "    def __init__(self, token_to_idx=None, unk_token=\"<UNK>\",\n",
    "                 mask_token=\"<MASK>\", begin_seq_token=\"<BEGIN>\",\n",
    "                 end_seq_token=\"<END>\"):\n",
    "\n",
    "        super(SequenceVocabulary, self).__init__(token_to_idx)\n",
    "\n",
    "        self._mask_token = mask_token\n",
    "        self._unk_token = unk_token\n",
    "        self._begin_seq_token = begin_seq_token\n",
    "        self._end_seq_token = end_seq_token\n",
    "\n",
    "        self.mask_index = self.add_token(self._mask_token)\n",
    "        self.unk_index = self.add_token(self._unk_token)\n",
    "        self.begin_seq_index = self.add_token(self._begin_seq_token)\n",
    "        self.end_seq_index = self.add_token(self._end_seq_token)\n",
    "\n",
    "    def lookup_token(self, token):\n",
    "        \"\"\"Retrieve the index associated with the token \n",
    "          or the UNK index if token isn't present.\n",
    "        \n",
    "        Args:\n",
    "            token (str): the token to look up \n",
    "        Returns:\n",
    "            index (int): the index corresponding to the token\n",
    "        Notes:\n",
    "            `unk_index` needs to be >=0 (having been added into the Vocabulary) \n",
    "              for the UNK functionality \n",
    "        \"\"\"\n",
    "        if self.unk_index >= 0:\n",
    "            return self._token_to_idx.get(token, self.unk_index)\n",
    "        else:\n",
    "            return self._token_to_idx[token]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReviewVectorizer(object):\n",
    "    \"\"\" The Vectorizer which coordinates the Vocabularies and puts them to use\"\"\"   \n",
    "    def __init__(self, review_vocab, rating_vocab, max_sequnce_length):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            review_vocab (Vocabulary): maps words to integers\n",
    "            rating_vocab (Vocabulary): maps class labels to integers; {'negative':0, 'positive':1}\n",
    "        \"\"\"\n",
    "        self.review_vocab = review_vocab\n",
    "        self.rating_vocab = rating_vocab\n",
    "        self.max_sequnce_length = max_sequnce_length\n",
    "\n",
    "    def vectorize(self, review, vector_length=-1):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            review (str): the string of words\n",
    "            vector_length (int): an argument for forcing the length of index vector\n",
    "        \"\"\"\n",
    "        indices = [self.review_vocab.begin_seq_index]       \n",
    "                \n",
    "        for token in review.split(\" \"):\n",
    "#             token = re.sub(r'(?!<\\d)\\.(?!\\d)|[^\\s\\w.]', '', token.lower()) #added regex\n",
    "#             token = wnl.lemmatize(contractions.fix(token))\n",
    "            if token not in string.punctuation:\n",
    "                indices.append(self.review_vocab.lookup_token(token))\n",
    "\n",
    "        indices.append(self.review_vocab.end_seq_index)\n",
    "\n",
    "        if vector_length < 0:\n",
    "            vector_length = len(indices)\n",
    "\n",
    "        out_vector = np.zeros(vector_length, dtype=np.int64)         \n",
    "        out_vector[:len(indices)] = indices\n",
    "        out_vector[len(indices):] = self.review_vocab.mask_index\n",
    "        \n",
    "        return out_vector, len(indices)\n",
    "\n",
    "    @classmethod\n",
    "    def from_dataframe(cls, review_df, cutoff=2):\n",
    "        \"\"\"Instantiate the vectorizer from the dataset dataframe\n",
    "        \n",
    "        Args:\n",
    "            review_df (pandas.DataFrame): the surnames dataset\n",
    "        Returns:\n",
    "            an instance of the ReviewVectorizer\n",
    "        \"\"\"\n",
    "        review_vocab = SequenceVocabulary()\n",
    "        rating_vocab = Vocabulary()\n",
    "        \n",
    "        max_sequnce_length = 0\n",
    "        \n",
    "        # Add ratings\n",
    "        for rating in sorted(set(review_df.rating)):\n",
    "            rating_vocab.add_token(rating)\n",
    "\n",
    "        \n",
    "        # Add top words if count > provided count\n",
    "        word_counts = Counter()\n",
    "        for review in review_df.review:\n",
    "            tokens = review.split(\" \")\n",
    "            if len(tokens) > max_sequnce_length:\n",
    "                max_sequnce_length = len(tokens) \n",
    "            for word in tokens:\n",
    "                if word not in string.punctuation:\n",
    "                    word_counts[word] += 1\n",
    "#                     word = re.sub(r'(?!<\\d)\\.(?!\\d)|[^\\s\\w.]', '', word.lower()) #added regex\n",
    "#                     word = wnl.lemmatize(contractions.fix(word))\n",
    "#                     word_counts[word] += 1 #added\n",
    "        #added\n",
    "        #word_counts = {word: count for word, count in word_counts.items() if word not in stopwords.words('english')}\n",
    "        for word, count in word_counts.items():\n",
    "            if count > cutoff:\n",
    "                review_vocab.add_token(word)\n",
    "\n",
    "        return cls(review_vocab, rating_vocab, max_sequnce_length)   \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReviewDataset(Dataset):\n",
    "    def __init__(self, review_df, vectorizer):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            review_df (pandas.DataFrame): the dataset\n",
    "            vectorizer (ReviewVectorizer): vectorizer instatiated from dataset\n",
    "        \"\"\"\n",
    "        self.review_df = review_df \n",
    "        self._vectorizer = vectorizer\n",
    "\n",
    "        self._max_seq_length = self._vectorizer.max_sequnce_length + 2   # add 2 for begin_seq_token and end_seq_token\n",
    "\n",
    "        self.train_df = self.review_df[self.review_df.split=='train']\n",
    "        self.train_size = len(self.train_df)\n",
    "\n",
    "        self.val_df = self.review_df[self.review_df.split=='val']\n",
    "        self.validation_size = len(self.val_df)\n",
    "\n",
    "        self.test_df = self.review_df[self.review_df.split=='test']\n",
    "        self.test_size = len(self.test_df)\n",
    "\n",
    "        self._lookup_dict = {'train': (self.train_df, self.train_size), \n",
    "                             'val': (self.val_df, self.validation_size), \n",
    "                             'test': (self.test_df, self.test_size)}\n",
    "\n",
    "        self.set_split('train')\n",
    "        \n",
    "        # Class weights\n",
    "        class_counts = self.train_df.rating.value_counts().to_dict()\n",
    "        def sort_key(item):\n",
    "            return self._vectorizer.rating_vocab.lookup_token(item[0])\n",
    "        sorted_counts = sorted(class_counts.items(), key=sort_key)\n",
    "        frequencies = [count for _, count in sorted_counts]\n",
    "        self.class_weights = 1.0 / torch.tensor(frequencies, dtype=torch.float32)\n",
    "\n",
    "        \n",
    "    @classmethod\n",
    "    def load_dataset_and_make_vectorizer(cls, review_csv):\n",
    "        \"\"\"Load dataset and make a new vectorizer from scratch\n",
    "        \n",
    "        Args:\n",
    "            review_csv (str): location of the dataset\n",
    "        Returns:\n",
    "            an instance of ReviewDataset\n",
    "        \"\"\"\n",
    "        review_df = pd.read_csv(review_csv)\n",
    "        train_review_df = review_df[review_df.split=='train']\n",
    "        return cls(review_df, ReviewVectorizer.from_dataframe(train_review_df))\n",
    "        \n",
    "    def get_vectorizer(self):\n",
    "        \"\"\" returns the vectorizer \"\"\"\n",
    "        return self._vectorizer\n",
    "\n",
    "    def set_split(self, split=\"train\"):\n",
    "        self._target_split = split\n",
    "        self._target_df, self._target_size = self._lookup_dict[split]\n",
    "\n",
    "    def __len__(self):\n",
    "        return self._target_size\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        \"\"\"the primary entry point method for PyTorch datasets\n",
    "        \n",
    "        Args:\n",
    "            index (int): the index to the data point \n",
    "        Returns:\n",
    "            a dictionary holding the data point's:\n",
    "                features (x_data)\n",
    "                label (y_target)\n",
    "                feature length (x_length)\n",
    "        \"\"\"\n",
    "        row = self._target_df.iloc[index]\n",
    "        \n",
    "        review_vector, vec_length = \\\n",
    "            self._vectorizer.vectorize(row.review, self._max_seq_length)\n",
    "        \n",
    "        rating_index = \\\n",
    "            self._vectorizer.rating_vocab.lookup_token(row.rating)\n",
    "\n",
    "        return {'x_data': review_vector, \n",
    "                'y_target': rating_index, \n",
    "                'x_length': vec_length}\n",
    "\n",
    "    def get_num_batches(self, batch_size):\n",
    "        \"\"\"Given a batch size, return the number of batches in the dataset\n",
    "        \n",
    "        Args:\n",
    "            batch_size (int)\n",
    "        Returns:\n",
    "            number of batches in the dataset\n",
    "        \"\"\"\n",
    "        return len(self) // batch_size\n",
    "\n",
    "def generate_batches(dataset, batch_size, shuffle=True,\n",
    "                     drop_last=False, device=\"cpu\"): \n",
    "    \"\"\"\n",
    "    A generator function which wraps the PyTorch DataLoader. It will \n",
    "      ensure each tensor is on the write device location.\n",
    "    \"\"\"\n",
    "    dataloader = DataLoader(dataset=dataset, batch_size=batch_size,\n",
    "                            shuffle=shuffle, drop_last=drop_last)\n",
    "\n",
    "    for data_dict in dataloader:\n",
    "        out_data_dict = {}\n",
    "        for name, tensor in data_dict.items():\n",
    "            out_data_dict[name] = data_dict[name].to(device)\n",
    "        yield out_data_dict\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def column_gather(y_out, x_lengths):\n",
    "    '''Get a specific vector from each batch datapoint in `y_out`.\n",
    "\n",
    "    More precisely, iterate over batch row indices, get the vector that's at\n",
    "    the position indicated by the corresponding value in `x_lengths` at the row\n",
    "    index.\n",
    "\n",
    "    Args:\n",
    "        y_out (torch.FloatTensor, torch.cuda.FloatTensor)\n",
    "            shape: (batch, sequence, feature)\n",
    "        x_lengths (torch.LongTensor, torch.cuda.LongTensor)\n",
    "            shape: (batch,)\n",
    "\n",
    "    Returns:\n",
    "        y_out (torch.FloatTensor, torch.cuda.FloatTensor)\n",
    "            shape: (batch, feature)\n",
    "    '''\n",
    "    x_lengths = x_lengths.long().detach().cpu().numpy() - 1\n",
    "\n",
    "    out = []\n",
    "    for batch_index, column_index in enumerate(x_lengths):\n",
    "        out.append(y_out[batch_index, column_index])\n",
    "\n",
    "    return torch.stack(out)\n",
    "\n",
    "def column_summation(y_out, x_lengths, mode=\"mean\"):\n",
    "    '''Get a max or mean vector from each batch datapoint in `y_out`.\n",
    "\n",
    "    More precisely, iterate over batch row indices, get the max or mean vector of all the vectors by \n",
    "    the position indicated by the corresponding value in `x_lengths` at the row index.\n",
    "\n",
    "    Args:\n",
    "        y_out (torch.FloatTensor, torch.cuda.FloatTensor)\n",
    "            shape: (batch, sequence, feature)\n",
    "        x_lengths (torch.LongTensor, torch.cuda.LongTensor)\n",
    "            shape: (batch,)\n",
    "        mode: \"mean\" for mean vector; \"max\" for max vector\n",
    "\n",
    "    Returns:\n",
    "        y_out (torch.FloatTensor, torch.cuda.FloatTensor)\n",
    "            shape: (batch, feature)\n",
    "    '''\n",
    "    x_lengths = x_lengths.long().detach().cpu().numpy() - 1\n",
    "\n",
    "    out = []\n",
    "    for batch_index, column_index in enumerate(x_lengths):\n",
    "        if mode == \"mean\":\n",
    "            out.append(y_out[batch_index, 0:column_index+1].mean(dim=0))  # get the mean vector\n",
    "        else:\n",
    "            out.append(y_out[batch_index, 0:column_index+1].max(dim=0).values)  # get the max vector\n",
    "\n",
    "    return torch.stack(out)\n",
    "\n",
    "\n",
    "class ReviewClassifier(nn.Module):\n",
    "    \"\"\" A Classifier with an RNN to extract features and an MLP to classify \"\"\"\n",
    "    def __init__(self, embedding_size, num_embeddings, num_classes,\n",
    "                 rnn_hidden_size, bidirectional=False, batch_first=True, padding_idx=0):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            embedding_size (int): The size of the word embeddings\n",
    "            num_embeddings (int): The number of words to embed\n",
    "            num_classes (int): The size of the prediction vector \n",
    "                Note: the number of sentiment polarities\n",
    "            bidirectional (bool): Informs whether bidrectional RNN is used\n",
    "            rnn_hidden_size (int): The size of the RNN's hidden state\n",
    "            batch_first (bool): Informs whether the input tensors will \n",
    "                have batch or the sequence on the 0th dimension\n",
    "            padding_idx (int): The index for the tensor padding; \n",
    "                see torch.nn.Embedding\n",
    "        \"\"\"\n",
    "        super(ReviewClassifier, self).__init__()\n",
    "\n",
    "        if bidirectional == False:\n",
    "             self.num_directions = 1\n",
    "        else:\n",
    "             self.num_directions = 2\n",
    "        \n",
    "        self.emb = nn.Embedding(num_embeddings=num_embeddings,\n",
    "                                embedding_dim=embedding_size,\n",
    "                                padding_idx=padding_idx)\n",
    "\n",
    "        #self.rnn = nn.RNN(input_size=embedding_size,\n",
    "        self.rnn = nn.GRU(input_size=embedding_size,\n",
    "        #self.rnn = nn.LSTM(input_size=embedding_size,\n",
    "                             hidden_size=rnn_hidden_size,\n",
    "                             batch_first=batch_first, \n",
    "                             num_layers = 1,\n",
    "                             dropout = 0.0, \n",
    "                             bidirectional=bidirectional)\n",
    "        \n",
    "        #self.fc1 = nn.Linear(in_features=rnn_hidden_size*self.num_directions,\n",
    "        #                 out_features=rnn_hidden_size*self.num_directions)\n",
    "        self.fc2 = nn.Linear(in_features=rnn_hidden_size*self.num_directions,\n",
    "                          out_features=num_classes)\n",
    "        # for batch norm\n",
    "        self.bn1 = nn.BatchNorm1d(rnn_hidden_size*self.num_directions) \n",
    "\n",
    "\n",
    "    def forward(self, x_in, x_lengths=None, apply_softmax=False):\n",
    "        \"\"\"The forward pass of the classifier\n",
    "        \n",
    "        Args:\n",
    "            x_in (torch.Tensor): an input data tensor. \n",
    "                x_in.shape should be (batch, input_dim)\n",
    "            x_lengths (torch.Tensor): the lengths of each sequence in the batch.\n",
    "                They are used to find the final vector of each sequence\n",
    "            apply_softmax (bool): a flag for the softmax activation\n",
    "                should be false if used with the Cross Entropy losses\n",
    "        Returns:\n",
    "            the resulting tensor. tensor.shape should be (batch, output_dim)\n",
    "        \"\"\"\n",
    "        x_embedded = self.emb(x_in) # x_embedded: (batch, seq_size, feat_size)\n",
    "               \n",
    "        # create PackedSequence; x_packed.data.shape=(number_items, embeddign_size)\n",
    "        #x_packed = pack_padded_sequence(x_embedded, x_lengths.detach().cpu().numpy(), batch_first=True)\n",
    "        y_out, _ = self.rnn(x_embedded)        \n",
    "        #y_out, _ = pad_packed_sequence(y_out, batch_first=True) # y_out: (batch, seq_size, hidden_size*num_directions)\n",
    "        \n",
    "        if x_lengths is not None:\n",
    "             y_out = column_gather(y_out, x_lengths)   # y_out gets the last hidden vector of each input: (batch, hidden_size*num_directions)\n",
    "             #y_out = column_summation(y_out, x_lengths, mode=\"max\") # y_out gets the max or mean hidden vector of each input\n",
    "        else:\n",
    "             y_out = y_out[:, -1, :]\n",
    "\n",
    "        \n",
    "        # with batch norm and dropout\n",
    "        #y_out = F.relu(self.bn1(self.fc1(F.dropout(y_out, 0.5, training=self.training))))\n",
    "        #y_out = F.relu(self.bn1(self.fc1(y_out)))\n",
    "\n",
    "        \n",
    "        # with dropout\n",
    "        #y_out = self.fc2(F.dropout(y_out, 0.4, training=self.training))          # y_out: (batch, num_classes)\n",
    "        y_out = self.fc2(y_out)\n",
    "\n",
    "        if apply_softmax:\n",
    "            y_out = F.softmax(y_out, dim=1)\n",
    "\n",
    "        return y_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_seed_everywhere(seed, cuda):\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    if cuda:\n",
    "        torch.cuda.manual_seed_all(seed)\n",
    "\n",
    "def handle_dirs(dirpath):\n",
    "    if not os.path.exists(dirpath):\n",
    "        os.makedirs(dirpath)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "code_folding": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using CUDA: False\n"
     ]
    }
   ],
   "source": [
    "args = Namespace(\n",
    "    # Data and path information\n",
    "    review_csv=\"data/rt-polaritydata/reviews_with_splits_lite.csv\",\n",
    "    model_state_file=\"model.pth\",\n",
    "    save_dir=\"model_storage/rt-polaritydata/review_classification\",\n",
    "    # Model hyper parameter\n",
    "    word_embedding_size=350,#100,\n",
    "    rnn_hidden_size=900,#100,\n",
    "    bidirectional=False,\n",
    "    #bidirectional=True,\n",
    "    # Training hyper parameter\n",
    "    num_epochs=100,\n",
    "    learning_rate=1e-3,\n",
    "    batch_size=32,#64,\n",
    "    seed=1337,\n",
    "    early_stopping_criteria=5,\n",
    "    # Runtime hyper parameter\n",
    "    cuda=True,\n",
    "    catch_keyboard_interrupt=True,\n",
    "    reload_from_files=False,\n",
    "    expand_filepaths_to_save_dir=True,\n",
    ")\n",
    "\n",
    "# Check CUDA\n",
    "if not torch.cuda.is_available():\n",
    "    args.cuda = False\n",
    "\n",
    "args.device = torch.device(\"cuda\" if args.cuda else \"cpu\")\n",
    "    \n",
    "print(\"Using CUDA: {}\".format(args.cuda))\n",
    "\n",
    "\n",
    "if args.expand_filepaths_to_save_dir:\n",
    "    args.model_state_file = os.path.join(args.save_dir,\n",
    "                                         args.model_state_file)\n",
    "    \n",
    "# Set seed for reproducibility\n",
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "\n",
    "# handle dirs\n",
    "handle_dirs(args.save_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "code_folding": [
     0,
     4
    ]
   },
   "outputs": [],
   "source": [
    "# create dataset and vectorizer\n",
    "dataset = ReviewDataset.load_dataset_and_make_vectorizer(args.review_csv)\n",
    "\n",
    "vectorizer = dataset.get_vectorizer()\n",
    "\n",
    "classifier = ReviewClassifier(embedding_size=args.word_embedding_size, \n",
    "                               num_embeddings=len(vectorizer.review_vocab),\n",
    "                               num_classes=len(vectorizer.rating_vocab),\n",
    "                               rnn_hidden_size=args.rnn_hidden_size,\n",
    "                               padding_idx=vectorizer.review_vocab.mask_index,\n",
    "                               bidirectional=args.bidirectional)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5614"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vectorizer.review_vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training Routine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "def make_train_state(args):\n",
    "    return {'stop_early': False,\n",
    "            'early_stopping_step': 0,\n",
    "            'early_stopping_best_val': 1e8,\n",
    "            'learning_rate': args.learning_rate,\n",
    "            'epoch_index': 0,\n",
    "            'train_loss': [],\n",
    "            'train_acc': [],\n",
    "            'val_loss': [],\n",
    "            'val_acc': [],\n",
    "            'test_loss': -1,\n",
    "            'test_acc': -1,\n",
    "            'model_filename': args.model_state_file}\n",
    "\n",
    "\n",
    "def update_train_state(args, model, train_state):\n",
    "    \"\"\"Handle the training state updates.\n",
    "\n",
    "    Components:\n",
    "     - Early Stopping: Prevent overfitting.\n",
    "     - Model Checkpoint: Model is saved if the model is better\n",
    "    \n",
    "    :param args: main arguments\n",
    "    :param model: model to train\n",
    "    :param train_state: a dictionary representing the training state values\n",
    "    :returns:\n",
    "        a new train_state\n",
    "    \"\"\"\n",
    "\n",
    "    # Save one model at least\n",
    "    if train_state['epoch_index'] == 0:\n",
    "        torch.save(model.state_dict(), train_state['model_filename'])\n",
    "        train_state['stop_early'] = False\n",
    "\n",
    "    # Save model if performance improved\n",
    "    elif train_state['epoch_index'] >= 1:\n",
    "        loss_tm1, loss_t = train_state['val_loss'][-2:]\n",
    "         \n",
    "        # If loss worsened\n",
    "        if loss_t >= loss_tm1:\n",
    "            # Update step\n",
    "            train_state['early_stopping_step'] += 1\n",
    "        # Loss decreased\n",
    "        else:\n",
    "            # Save the best model\n",
    "            if loss_t < train_state['early_stopping_best_val']:\n",
    "                torch.save(model.state_dict(), train_state['model_filename'])\n",
    "                train_state['early_stopping_best_val'] = loss_t\n",
    "\n",
    "            # Reset early stopping step\n",
    "            train_state['early_stopping_step'] = 0\n",
    "\n",
    "        # Stop early ?\n",
    "        train_state['stop_early'] = \\\n",
    "            train_state['early_stopping_step'] >= args.early_stopping_criteria\n",
    "\n",
    "    return train_state\n",
    "\n",
    "\n",
    "def compute_accuracy(y_pred, y_target):\n",
    "    _, y_pred_indices = y_pred.max(dim=1)\n",
    "    n_correct = torch.eq(y_pred_indices, y_target).sum().item()\n",
    "    return n_correct / len(y_pred_indices) * 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "code_folding": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f1ba6aeb039942dab65b8a2c063c7ac5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "training routine:   0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b1ada8202871416b9446e771e2d93dbc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "split=train:   0%|          | 0/233 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "509fccab6a034e169beecda9e605fb4d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "split=val:   0%|          | 0/49 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "classifier = classifier.to(args.device)\n",
    "dataset.class_weights = dataset.class_weights.to(args.device)\n",
    "    \n",
    "loss_func = nn.CrossEntropyLoss(dataset.class_weights)\n",
    "optimizer = optim.Adam(classifier.parameters(), lr=args.learning_rate)\n",
    "scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer=optimizer, mode='min', factor=0.1, patience=10)\n",
    "\n",
    "train_state = make_train_state(args)\n",
    "\n",
    "epoch_bar = tqdm(desc='training routine', total=args.num_epochs, position=0)\n",
    "\n",
    "dataset.set_split('train')\n",
    "train_bar = tqdm(desc='split=train', total=dataset.get_num_batches(args.batch_size), position=1, leave=True)\n",
    "\n",
    "dataset.set_split('val')\n",
    "val_bar = tqdm(desc='split=val', total=dataset.get_num_batches(args.batch_size), position=1, leave=True)\n",
    "\n",
    "try:\n",
    "    for epoch_index in range(args.num_epochs):\n",
    "        train_state['epoch_index'] = epoch_index\n",
    "\n",
    "        # Iterate over training dataset\n",
    "\n",
    "        # setup: batch generator, set loss and acc to 0, set train mode on\n",
    "        dataset.set_split('train')\n",
    "        batch_generator = generate_batches(dataset, \n",
    "                                           batch_size=args.batch_size, \n",
    "                                           device=args.device)\n",
    "        running_loss = 0.0\n",
    "        running_acc = 0.0\n",
    "        classifier.train()\n",
    "\n",
    "        for batch_index, batch_dict in enumerate(batch_generator):\n",
    "            # the training routine is these 5 steps:\n",
    "\n",
    "            # --------------------------------------    \n",
    "            # step 1. zero the gradients\n",
    "            optimizer.zero_grad()\n",
    "           \n",
    "            # step 2. compute the output\n",
    "            y_pred = classifier(x_in=batch_dict['x_data'], \n",
    "                                x_lengths=batch_dict['x_length'])\n",
    "\n",
    "            # step 3. compute the loss\n",
    "            loss = loss_func(y_pred, batch_dict['y_target'])\n",
    "    \n",
    "            running_loss += (loss.item() - running_loss) / (batch_index + 1)\n",
    "\n",
    "            # step 4. use loss to produce gradients\n",
    "            loss.backward()\n",
    "\n",
    "            # step 5. use optimizer to take gradient step\n",
    "            optimizer.step()\n",
    "            # -----------------------------------------\n",
    "            # compute the accuracy\n",
    "            acc_t = compute_accuracy(y_pred, batch_dict['y_target'])\n",
    "            running_acc += (acc_t - running_acc) / (batch_index + 1)\n",
    "\n",
    "            # update bar\n",
    "            train_bar.set_postfix(loss=running_loss, acc=running_acc, epoch=epoch_index)\n",
    "            train_bar.update()\n",
    "\n",
    "        train_state['train_loss'].append(running_loss)\n",
    "        train_state['train_acc'].append(running_acc)\n",
    "\n",
    "        # Iterate over val dataset\n",
    "\n",
    "        # setup: batch generator, set loss and acc to 0; set eval mode on\n",
    "\n",
    "        dataset.set_split('val')\n",
    "        batch_generator = generate_batches(dataset, \n",
    "                                           batch_size=args.batch_size, \n",
    "                                           device=args.device)\n",
    "        running_loss = 0.\n",
    "        running_acc = 0.\n",
    "        classifier.eval()\n",
    "\n",
    "        for batch_index, batch_dict in enumerate(batch_generator):\n",
    "            # compute the output\n",
    "            y_pred = classifier(x_in=batch_dict['x_data'], \n",
    "                                x_lengths=batch_dict['x_length'])\n",
    "\n",
    "            # step 3. compute the loss\n",
    "            loss = loss_func(y_pred, batch_dict['y_target'])\n",
    "            running_loss += (loss.item() - running_loss) / (batch_index + 1)\n",
    "\n",
    "            # compute the accuracy\n",
    "            acc_t = compute_accuracy(y_pred, batch_dict['y_target'])\n",
    "            running_acc += (acc_t - running_acc) / (batch_index + 1)\n",
    "            val_bar.set_postfix(loss=running_loss, acc=running_acc, epoch=epoch_index)\n",
    "            val_bar.update()\n",
    "\n",
    "        train_state['val_loss'].append(running_loss)\n",
    "        train_state['val_acc'].append(running_acc)\n",
    "\n",
    "        train_state = update_train_state(args=args, model=classifier, \n",
    "                                         train_state=train_state)\n",
    "\n",
    "        scheduler.step(train_state['val_loss'][-1])\n",
    "\n",
    "        train_bar.n = 0\n",
    "        val_bar.n = 0\n",
    "        epoch_bar.update()\n",
    "\n",
    "        if train_state['stop_early']:\n",
    "            break\n",
    "            \n",
    "except KeyboardInterrupt:\n",
    "    print(\"Exiting loop\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHFCAYAAAAOmtghAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAA9hAAAPYQGoP6dpAABcIElEQVR4nO3de3zO9f/H8ce1jc2wYezUGHI+n0rI+ThyaIkcMqHyjRB9Kymn1L71/ZJOSIUICSNyzvlQOWRIS8oxthzbHIft8/vj89vFZcPM7HPt2vN+u123+ryvz+f6vDbLnr3fn/f7bTMMw0BERETERbhZXYCIiIhIZlK4EREREZeicCMiIiIuReFGREREXIrCjYiIiLgUhRsRERFxKQo3IiIi4lIUbkRERMSlKNyIiIiIS1G4EbmBzWZL12vdunX3dJ+RI0dis9kydO26desypQZn17NnT4oXL+4U9y1evDg9e/a847X38mezZcsWRo4cyT///JPqvUaNGtGoUaO7/sx7dejQIWw2G9OmTcvye4vcCw+rCxBxJj/88IPD8VtvvcXatWtZs2aNQ3uFChXu6T59+vShVatWGbq2Ro0a/PDDD/dcg6TfggUL8PHxua/32LJlC6NGjaJnz54UKFDA4b0JEybc13uLuBqFG5EbPPLIIw7HRYoUwc3NLVX7zS5evIi3t3e67xMSEkJISEiGavTx8bljPZK5qlevbun9FWRF7o6GpUTuUqNGjahUqRIbNmygbt26eHt706tXLwDmzJlDixYtCAoKIk+ePJQvX57XXnuNCxcuOHxGWsNSxYsX57HHHmP58uXUqFGDPHnyUK5cOaZMmeJwXlpDHz179iRfvnz88ccftG7dmnz58lG0aFGGDBlCYmKiw/V//fUXHTt2JH/+/BQoUIBu3bqxbdu2dA0/nDx5khdeeIEKFSqQL18+/P39adKkCRs3bnQ4L2U443//+x/jxo2jRIkS5MuXjzp16vDjjz+m+txp06ZRtmxZPD09KV++PNOnT79tHSk6dOhAaGgoycnJqd6rXbs2NWrUsB9/8sknNGjQAH9/f/LmzUvlypV57733uHr16h3vk9aw1G+//UarVq3w9vamcOHC9O3bl3PnzqW6dtWqVbRv356QkBC8vLwoVaoUzz//PKdOnbKfM3LkSP79738DUKJEiVTDn2kNS505c4YXXniBBx54gNy5c1OyZEmGDRuW6s/bZrPRv39/ZsyYQfny5fH29qZq1ap89913d/y6b2XTpk00bdqU/Pnz4+3tTd26dVmyZInDORcvXuTll1+mRIkSeHl5UahQIWrVqsXs2bPt5xw4cICnnnqK4OBgPD09CQgIoGnTpkRHR2e4NhFQz41IhsTGxtK9e3deeeUV3nnnHdzczP9P2L9/P61bt2bQoEHkzZuX3377jXfffZetW7emGtpKy65duxgyZAivvfYaAQEBfP755/Tu3ZtSpUrRoEGD21579epV2rVrR+/evRkyZAgbNmzgrbfewtfXl+HDhwNw4cIFGjduzJkzZ3j33XcpVaoUy5cvp3Pnzun6us+cOQPAiBEjCAwM5Pz58yxYsIBGjRqxevXqVL+AP/nkE8qVK8f48eMBePPNN2ndujUHDx7E19cXMIPNM888Q/v27Rk7dizx8fGMHDmSxMRE+/f1Vnr16kX79u1Zs2YNzZo1s7f/9ttvbN26lQ8//NDe9ueff9K1a1dKlChB7ty52bVrF2+//Ta//fZbqgB5J3///TcNGzYkV65cTJgwgYCAAGbOnEn//v1Tnfvnn39Sp04d+vTpg6+vL4cOHWLcuHE8+uij7Nmzh1y5ctGnTx/OnDnDRx99RFRUFEFBQcCte2wuX75M48aN+fPPPxk1ahRVqlRh48aNREZGEh0dnSpoLFmyhG3btjF69Gjy5cvHe++9x+OPP86+ffsoWbLkXX3t69evp3nz5lSpUoUvvvgCT09PJkyYQNu2bZk9e7b9Z2nw4MHMmDGDMWPGUL16dS5cuMAvv/zC6dOn7Z/VunVrkpKSeO+99yhWrBinTp1iy5YtaT53JHJXDBG5pYiICCNv3rwObQ0bNjQAY/Xq1be9Njk52bh69aqxfv16AzB27dplf2/EiBHGzf/5hYaGGl5eXsbhw4ftbZcuXTIKFSpkPP/88/a2tWvXGoCxdu1ahzoB45tvvnH4zNatWxtly5a1H3/yyScGYCxbtszhvOeff94AjKlTp972a7rZtWvXjKtXrxpNmzY1Hn/8cXv7wYMHDcCoXLmyce3aNXv71q1bDcCYPXu2YRiGkZSUZAQHBxs1atQwkpOT7ecdOnTIyJUrlxEaGnrb+1+9etUICAgwunbt6tD+yiuvGLlz5zZOnTqV5nVJSUnG1atXjenTpxvu7u7GmTNn7O9FRESkum9oaKgRERFhP3711VcNm81mREdHO5zXvHnzVH82N0r5mTh8+LABGN9++639vf/+978GYBw8eDDVdQ0bNjQaNmxoP540aVKaf97vvvuuARgrV660twFGQECAkZCQYG+Li4sz3NzcjMjIyDTrTJHy53jjz8Ujjzxi+Pv7G+fOnbO3Xbt2zahUqZIREhJi/3OsVKmS0aFDh1t+9qlTpwzAGD9+/G1rEMkIDUuJZEDBggVp0qRJqvYDBw7QtWtXAgMDcXd3J1euXDRs2BCAmJiYO35utWrVKFasmP3Yy8uLMmXKcPjw4Ttea7PZaNu2rUNblSpVHK5dv349+fPnT/Uwc5cuXe74+SkmTZpEjRo18PLywsPDg1y5crF69eo0v742bdrg7u7uUA9gr2nfvn0cP36crl27OgzThYaGUrdu3TvW4uHhQffu3YmKiiI+Ph6ApKQkZsyYQfv27fHz87Ofu3PnTtq1a4efn5/9z6ZHjx4kJSXx+++/p/vrB1i7di0VK1akatWqDu1du3ZNde6JEyfo27cvRYsWtX+/QkNDgfT9TKRlzZo15M2bl44dOzq0pwydrV692qG9cePG5M+f334cEBCAv79/un6ubnThwgV++uknOnbsSL58+ezt7u7uPP300/z111/s27cPgIcffphly5bx2muvsW7dOi5duuTwWYUKFeLBBx/kv//9L+PGjWPnzp1pDi+KZITCjUgGpAwb3Oj8+fPUr1+fn376iTFjxrBu3Tq2bdtGVFQUQKq/3NNy4y/jFJ6enum61tvbGy8vr1TXXr582X58+vRpAgICUl2bVltaxo0bx7/+9S9q167N/Pnz+fHHH9m2bRutWrVKs8abvx5PT0/g+vciZYgiMDAw1bVptaWlV69eXL58ma+//hqAFStWEBsbyzPPPGM/58iRI9SvX59jx47xwQcfsHHjRrZt28Ynn3ziUE96nT59Ol01Jycn06JFC6KionjllVdYvXo1W7dutT93dLf3vfn+Nz+35e/vj4eHh8PQD9zbz9WNzp49i2EYaf78BwcH22sD+PDDD3n11VdZuHAhjRs3plChQnTo0IH9+/cDZhhfvXo1LVu25L333qNGjRoUKVKEAQMGpPnsksjd0DM3IhmQ1ho1a9as4fjx46xbt87eWwM41fMDfn5+bN26NVV7XFxcuq7/6quvaNSoERMnTnRoz+gvo5RfumndP701VahQgYcffpipU6fy/PPPM3XqVIKDg2nRooX9nIULF3LhwgWioqLsvSZAhh9c9fPzS1fNv/zyC7t27WLatGlERETY2//4448M3ffG+//0008YhuHws3jixAmuXbtG4cKF7+nzb6VgwYK4ubkRGxub6r3jx48D2O+dN29eRo0axahRo/j777/tvTht27blt99+A8weui+++AKA33//nW+++YaRI0dy5coVJk2adF++BskZ1HMjkklSfsmk9E6k+PTTT60oJ00NGzbk3LlzLFu2zKE9pdfjTmw2W6qvb/fu3anWB0qvsmXLEhQUxOzZszEMw95++PBhtmzZku7PeeaZZ/jpp5/YtGkTixcvJiIiwmE4LK0/G8Mw+OyzzzJUd+PGjdm7dy+7du1yaJ81a5bD8d38TNzcq3U7TZs25fz58yxcuNChPWWWWdOmTe/4GRmRN29eateuTVRUlEOdycnJfPXVV4SEhFCmTJlU1wUEBNCzZ0+6dOnCvn37uHjxYqpzypQpwxtvvEHlypX5+eef70v9knOo50Ykk9StW5eCBQvSt29fRowYQa5cuZg5c2aqX4BWioiI4P3336d79+6MGTOGUqVKsWzZMlasWAFwx9lJjz32GG+99RYjRoygYcOG7Nu3j9GjR1OiRAmuXbt21/W4ubnx1ltv0adPHx5//HGeffZZ/vnnH0aOHJnuYSkwnxkaPHgwXbp0ITExMdW07ebNm5M7d266dOnCK6+8wuXLl5k4cSJnz56965oBBg0axJQpU2jTpg1jxoyxz5ZK6ZFIUa5cOR588EFee+01DMOgUKFCLF68mFWrVqX6zMqVKwPwwQcfEBERQa5cuShbtqzDszIpevTowSeffEJERASHDh2icuXKbNq0iXfeeYfWrVs7zBzLbJGRkTRv3pzGjRvz8ssvkzt3biZMmMAvv/zC7Nmz7YGudu3aPPbYY1SpUoWCBQsSExPDjBkzqFOnDt7e3uzevZv+/fvz5JNPUrp0aXLnzs2aNWvYvXs3r7322n2rX3IG9dyIZBI/Pz+WLFmCt7c33bt3p1evXuTLl485c+ZYXZpd3rx5WbNmDY0aNeKVV17hiSee4MiRI/YVcG9eGfdmw4YNY8iQIXzxxRe0adOGzz//nEmTJvHoo49muKbevXvz+eef8+uvvxIeHs7o0aN5/fXX03xg+1Z8fX15/PHH+euvv6hXr16q3oNy5coxf/58zp49S3h4OC+++CLVqlVzmCp+NwIDA1m/fj0VKlTgX//6F927d8fLy4uPP/7Y4bxcuXKxePFiypQpw/PPP0+XLl04ceIE33//farPbNSoEUOHDmXx4sU8+uijPPTQQ+zYsSPN+3t5ebF27Vq6devGf//7X8LCwpg2bRovv/yy/Rmv+6Vhw4b2B5p79uzJU089RXx8PIsWLXJYUqBJkyYsWrSIZ555hhYtWvDee+/Ro0cPFi9eDJjfwwcffJAJEybQsWNH2rdvz+LFixk7diyjR4++r1+DuD6bcWNfsIjkSO+88w5vvPEGR44cyfDKySIizkLDUiI5TErvQrly5bh69Spr1qzhww8/pHv37go2IuISFG5Echhvb2/ef/99Dh06RGJiIsWKFePVV1/ljTfesLo0EZFMoWEpERERcSl6oFhERERcisKNiIiIuBSFGxEREXEpOe6B4uTkZI4fP07+/PnTXEJfREREnI9hGJw7d47g4OA7Ljia48LN8ePHKVq0qNVliIiISAYcPXr0jstW5Lhwk7KU+dGjR/Hx8bG4GhEREUmPhIQEihYtmuaWJDfLceEmZSjKx8dH4UZERCSbSc8jJXqgWERERFyKwo2IiIi4FIUbERERcSk57pmb9EpKSuLq1atWlyHZXK5cuXB3d7e6DBGRHEXh5iaGYRAXF8c///xjdSniIgoUKEBgYKDWVRIRySKWhpsNGzbw3//+lx07dhAbG8uCBQvo0KFDuq7dvHkzDRs2pFKlSkRHR2daTSnBxt/fH29vb/1CkgwzDIOLFy9y4sQJAIKCgiyuSEQkZ7A03Fy4cIGqVavyzDPP8MQTT6T7uvj4eHr06EHTpk35+++/M62epKQke7Dx8/PLtM+VnCtPnjwAnDhxAn9/fw1RiYhkAUvDTVhYGGFhYXd93fPPP0/Xrl1xd3dn4cKFmVZPyjM23t7emfaZIik/T1evXlW4ERHJAtluttTUqVP5888/GTFiRLrOT0xMJCEhweF1JxqKksyknycRkayVrcLN/v37ee2115g5cyYeHunrdIqMjMTX19f+0r5SIiIiri3bhJukpCS6du3KqFGjKFOmTLqvGzp0KPHx8fbX0aNH72OVrqVRo0YMGjQo3ecfOnQIm82WqQ94p2XdunXYbDbNaBMRkTRlm6ng586dY/v27ezcuZP+/fsDkJycjGEYeHh4sHLlSpo0aZLqOk9PTzw9PbO6XJKSYONGiI2FoCCoXx/u1+MWdxr2iIiIYNq0aXf9uVFRUeTKlSvd5xctWpTY2FgKFy581/cSERHJLNkm3Pj4+LBnzx6HtgkTJrBmzRrmzZtHiRIlLKostagoGDgQ/vrreltICHzwAYSHZ/79YmNj7f8+Z84chg8fzr59++xtKTN2Uly9ejVdoaVQoUJ3VYe7uzuBgYF3dY2IiLiWU6fM33/VqllXg6XDUufPnyc6Oto+jHHw4EGio6M5cuQIYA4p9ejRAwA3NzcqVark8PL398fLy4tKlSqRN29eq74MB1FR0LGjY7ABOHbMbI+Kyvx7BgYG2l++vr7YbDb78eXLlylQoADffPMNjRo1wsvLi6+++orTp0/TpUsXQkJC8Pb2pnLlysyePdvhc28elipevDjvvPMOvXr1In/+/BQrVozJkyfb3795WCpl+Gj16tXUqlULb29v6tat6xC8AMaMGYO/vz/58+enT58+vPbaa1S7y/8q5s+fT8WKFfH09KR48eKMHTvW4f0JEyZQunRpvLy8CAgIoGPHjvb35s2bR+XKlcmTJw9+fn40a9aMCxcu3NX9RURysn374H//gwYNICAAIiKsrcfScLN9+3aqV69O9erVARg8eDDVq1dn+PDhgNkjkRJ0soOkJLPHxjBSv5fSNmiQeV5We/XVVxkwYAAxMTG0bNmSy5cvU7NmTb777jt++eUXnnvuOZ5++ml++umn237O2LFjqVWrFjt37uSFF17gX//6F7/99tttrxk2bBhjx45l+/bteHh40KtXL/t7M2fO5O233+bdd99lx44dFCtWjIkTJ97V17Zjxw46derEU089xZ49exg5ciRvvvmmfShu+/btDBgwgNGjR7Nv3z6WL19OgwYNAPNnrEuXLvTq1YuYmBjWrVtHeHg4Rlp/iCIiAsC1a+ajF//+N5QtC+XKmf++cSMkJ5vnnD9vYYFGDhMfH28ARnx8fKr3Ll26ZPz666/GpUuXMvTZa9cahhljbv9au/bevobbmTp1quHr62s/PnjwoAEY48ePv+O1rVu3NoYMGWI/btiwoTFw4ED7cWhoqNG9e3f7cXJysuHv729MnDjR4V47d+40DMMw1q5dawDG999/b79myZIlBmD/HteuXdvo16+fQx316tUzqlatess6Uz737NmzhmEYRteuXY3mzZs7nPPvf//bqFChgmEYhjF//nzDx8fHSEhISPVZO3bsMADj0KFDt7zfvbrXnysREWeQkGAYc+caRo8ehuHn5/h7LVcuw2je3DA++sgw7tdfp7f7/X2zbDNbKju44dGXTDkvM9WqVcvhOCkpibfffpsqVarg5+dHvnz5WLly5R17yqpUqWL/95Thr5TtBdJzTcoWBCnX7Nu3j4cfftjh/JuP7yQmJoZ69eo5tNWrV4/9+/eTlJRE8+bNCQ0NpWTJkjz99NPMnDmTixcvAlC1alWaNm1K5cqVefLJJ/nss884e/bsXd1fRMRVHT0KEyZAq1ZQuDA8+SRMnw6nT0PBgtC9O8yZAydPwsqV0L8/hIZaXXU2mgqeHaR36yArthi6+ZmksWPH8v777/PKK6+wZs0aoqOjadmyJVeuXLnt59z8ILLNZiM5pQ8yHdekzOy68ZqbZ3sZdzkkZBjGbT8jf/78/Pzzz8yePZugoCCGDx9O1apV+eeff3B3d2fVqlUsW7aMChUq8NFHH1G2bFkOHjx4VzWIiLgCw4AdO2DECKheHYoVg379YMUKuHIFSpWCwYNh3To4cQJmzIBOncDX1+rKHSncZKL69c1ZUbeamW2zQdGi5nlW27hxI+3bt6d79+5UrVqVkiVLsn///iyvo2zZsmzdutWhbfv27Xf1GRUqVGDTpk0ObVu2bKFMmTL27Q48PDxo1qwZ7733Hrt37+bQoUOsWbMGMMNVvXr1GDVqFDt37iR37twsWLDgHr4qEZHs4/JlWLoU/vUv83dUrVowejRER5u/t+rVg3ffhZgY+P13GDsWGjaEdK6lawknLi37cXc3p3t37Gj+QNzYAZESeMaPv3/r3dyNUqVKMX/+fLZs2ULBggUZN24ccXFxlC9fPkvrePHFF3n22WepVasWdevWZc6cOezevZuSJUum+zOGDBnCQw89xFtvvUXnzp354Ycf+Pjjj5kwYQIA3333HQcOHKBBgwYULFiQpUuXkpycTNmyZfnpp59YvXo1LVq0wN/fn59++omTJ09m+fdBRCQrnTwJS5bAokXmcNKNE0Tz5oWWLaFtW2jTBooUsa7OjFK4yWTh4TBvXtrr3Iwff3/WucmIN998k4MHD9KyZUu8vb157rnn6NChA/Hx8VlaR7du3Thw4AAvv/wyly9fplOnTvTs2TNVb87t1KhRg2+++Ybhw4fz1ltvERQUxOjRo+nZsycABQoUICoqipEjR3L58mVKly7N7NmzqVixIjExMWzYsIHx48eTkJBAaGgoY8eOzdCGriIizsow4LffzDCzaBH88IPj/4AHB0O7duarcWPw8rKu1sxgM+72AYdsLiEhAV9fX+Lj4/Hx8XF47/Llyxw8eJASJUrgdY9/slm5QrGrad68OYGBgcyYMcPqUjJFZv5ciYik17VrsGkTLF5sBpo//nB8v3r164GmevVbP1LhLG73+/tm6rm5T9zdoVEjq6twfhcvXmTSpEm0bNkSd3d3Zs+ezffff8+qVausLk1EJNuJjzcf/l20yHyO5sbJn7lzQ5Mm5nBT27bm8zWuSuFGLGWz2Vi6dCljxowhMTGRsmXLMn/+fJo1a2Z1aSIi2cKhQ2bvzOLF5iymq1evv+fnZz43064dtGgB+fNbVWXWUrgRS+XJk4fvv//e6jJERLKN5GRzunbK8zO7dzu+X7bs9eGmOnVy5iMRCjciIiJO7tIlWL3aDDOLF0Nc3PX33NzM6drt2pnDTWXLWlens1C4ERERcUJ//+04XfvSpevv5ctnrhrcrh20bm0OP8l1CjciIiJOwDDg11+vDzf99JPjdO2iRa8PNzVsCJ6e1tXq7BRuRERELHL1qrlsSEqguXnnl5o1rweaqlWdf7q2s1C4ERERyUL//APLl1+frn3j2qmentC0qRlmHnsMHnjAsjKzNYUbERGR++zAgeuL6W3YYC6wl6JIETPItGsHzZqZz9PIvdHGmWLXqFEjBg0aZD8uXrw448ePv+01NpuNhQsX3vO9M+tzbmfkyJFUq1btvt5DRATM6do//givvw6VKsGDD8KgQbBmjRlsypeHV1+FzZvNleynTIEOHRRsMot6blxA27ZtuXTpUprrxfzwww/UrVuXHTt2UKNGjbv63G3btpE3b97MKhMwA8bChQuJjo52aI+NjaVgwYKZei8Rkax08SJ8/73ZO/Pdd+ZspxTu7uY2PCnTtUuVsq7OnEDhxgX07t2b8PBwDh8+TGhoqMN7U6ZMoVq1ancdbACKZOFWsIGBgVl2LxGRzBIbawaZRYvMYHP58vX3fHwgLMwMNK1aQaFC1tWZ02hYygU89thj+Pv7M23aNIf2ixcvMmfOHHr37s3p06fp0qULISEheHt7U7lyZWbPnn3bz715WGr//v00aNAALy8vKlSokOb+T6+++iplypTB29ubkiVL8uabb3L1/9cCnzZtGqNGjWLXrl3YbDZsNpu95puHpfbs2UOTJk3IkycPfn5+PPfcc5w/f97+fs+ePenQoQP/+9//CAoKws/Pj379+tnvlR7JycmMHj2akJAQPD09qVatGsuXL7e/f+XKFfr3709QUBBeXl4UL16cyMhI+/sjR46kWLFieHp6EhwczIABA9J9bxHJngzDXBF4zBioXdvcTfu558yAc/kyhIbCiy/CqlVw8iR8/TV07apgk9XUc3MHhmF2NVrB2zt90/48PDzo0aMH06ZNY/jw4dj+/6K5c+dy5coVunXrxsWLF6lZsyavvvoqPj4+LFmyhKeffpqSJUtSu3btO94jOTmZ8PBwChcuzI8//khCQoLD8zkp8ufPz7Rp0wgODmbPnj08++yz5M+fn1deeYXOnTvzyy+/sHz5cvsQmq+vb6rPuHjxIq1ateKRRx5h27ZtnDhxgj59+tC/f3+HALd27VqCgoJYu3Ytf/zxB507d6ZatWo8++yzd/6mAR988AFjx47l008/pXr16kyZMoV27dqxd+9eSpcuzYcffsiiRYv45ptvKFasGEePHuXo0aMAzJs3j/fff5+vv/6aihUrEhcXx65du9J1XxHJXpKTYe1a+PZbs4fm8GHH9x9++Pp07UqVNF3bKRg5THx8vAEY8fHxqd67dOmS8euvvxqXLl2yt50/bxhmxMn61/nz6f+6YmJiDMBYs2aNva1BgwZGly5dbnlN69atjSFDhtiPGzZsaAwcONB+HBoaarz//vuGYRjGihUrDHd3d+Po0aP295ctW2YAxoIFC255j/fee8+oWbOm/XjEiBFG1apVU5134+dMnjzZKFiwoHH+hm/AkiVLDDc3NyMuLs4wDMOIiIgwQkNDjWvXrtnPefLJJ43OnTvfspab7x0cHGy8/fbbDuc89NBDxgsvvGAYhmG8+OKLRpMmTYzk5ORUnzV27FijTJkyxpUrV255vxRp/VyJiPOLizOMd94xjBIlHP9u9vIyjLZtDWPyZMM4ftzqKnOO2/3+vpmGpVxEuXLlqFu3LlOmTAHgzz//ZOPGjfTq1QuApKQk3n77bapUqYKfnx/58uVj5cqVHDlyJF2fHxMTQ7FixQgJCbG31alTJ9V58+bN49FHHyUwMJB8+fLx5ptvpvseN96ratWqDg8z16tXj+TkZPbt22dvq1ixIu437AgXFBTEiRMn0nWPhIQEjh8/Tr169Rza69WrR0xMDGAOfUVHR1O2bFkGDBjAypUr7ec9+eSTXLp0iZIlS/Lss8+yYMECrt04t1NEsiXDMGc0deoEISHmbKeDB6FAAejVy+y9OX3a7MF59lkICrK6YkmLws0deHvD+fPWvLy9767W3r17M3/+fBISEpg6dSqhoaE0bdoUgLFjx/L+++/zyiuvsGbNGqKjo2nZsiVXrlxJ12cbN64B/v9sN/W9/vjjjzz11FOEhYXx3XffsXPnToYNG5bue9x4r5s/O6175sqVK9V7ycnJd3Wvm+9z471r1KjBwYMHeeutt7h06RKdOnWiY8eOABQtWpR9+/bxySefkCdPHl544QUaNGhwV8/8iIjzOH0axo41N51s2hTmzjWnbD/yCEydCseOwRdfmENPd/t3s2Q9PXNzBzYbZPJs6PumU6dODBw4kFmzZvHll1/y7LPP2n9Rb9y4kfbt29O9e3fAfIZm//79lC9fPl2fXaFCBY4cOcLx48cJDg4GzGnmN9q8eTOhoaEMGzbM3nb4psHp3Llzk5SUdMd7ffnll1y4cMHee7N582bc3NwoU6ZMuuq9Ex8fH4KDg9m0aRMNGjSwt2/ZsoWHH37Y4bzOnTvTuXNnOnbsSKtWrThz5gyFChUiT548tGvXjnbt2tGvXz/KlSvHnj17MjQzTUSynmGY68xMmgTz5kFiotmePz907w7PP29ueSDZj8KNC8mXLx+dO3fm9ddfJz4+np49e9rfK1WqFPPnz2fLli0ULFiQcePGERcXl+5w06xZM8qWLUuPHj0YO3YsCQkJDiEm5R5Hjhzh66+/5qGHHmLJkiUsWLDA4ZzixYtz8OBBoqOjCQkJIX/+/HjetPtbt27dGDFiBBEREYwcOZKTJ0/y4osv8vTTTxMQEJCxb04a/v3vfzNixAgefPBBqlWrxtSpU4mOjmbmzJkAvP/++wQFBVGtWjXc3NyYO3cugYGBFChQgGnTppGUlETt2rXx9vZmxowZ5MmTJ9VUfBFxPv/8AzNmmKHm11+vt9eoAX37QpcuWkwvu9OwlIvp3bs3Z8+epVmzZhQrVsze/uabb1KjRg1atmxJo0aNCAwMpEOHDun+XDc3NxYsWEBiYiIPP/wwffr04e2333Y4p3379rz00kv079+fatWqsWXLFt58802Hc5544glatWpF48aNKVKkSJrT0b29vVmxYgVnzpzhoYceomPHjjRt2pSPP/747r4ZdzBgwACGDBnCkCFDqFy5MsuXL2fRokWULl0aMMPiu+++S61atXjooYc4dOgQS5cuxc3NjQIFCvDZZ59Rr149qlSpwurVq1m8eDF+fn6ZWqOIZA7DMHfZ7tXLnL49YIAZbLy9oXdv2LYNduwwn6NRsMn+bEZaD1O4sISEBHx9fYmPj8fHx8fhvcuXL3Pw4EFKlCiBl5eXRRWKq9HPlYh1zp2DWbPMXpobF0avVMnspeneHdJYkUKc0O1+f99Mw1IiIuJydu6ETz+FmTPNCRpg7rjdqZMZaurU0Xo0rkzhRkREXMLFizBnjtlLs3Xr9fYyZcxA06MHaOQ4Z1C4ERGRbG3vXrOXZvp0iI8323LlgvBwM9Q0bKhempxG4UZERLKdy5fN6duffgqbNl1vL1nS3OvpmWfA39+6+sRaCjdpyGHPWMt9pp8nkczz++9moJk2Dc6cMdvc3c3F9fr2hWbNwE3zgHM8hZsbpKx4e/HiRfLkyWNxNeIqLv7/zqs3r6gsIulz5QosXGg+S7N27fX2okXNqdu9e5vTu0VSKNzcwN3dnQIFCtj3J/L29r7lNgAid2IYBhcvXuTEiRMUKFDAYR8sEbmzgwdh8mSYMgVSto2z2aBNG3P14LAws9dG5GYKNzcJDAwESPcGjCJ3UqBAAfvPlYjc3rVr8N13Zi/NypXm4ntgblDZp4/5umF9UpE0KdzcxGazERQUhL+/vzZBlHuWK1cu9diIpMPRo/D55+br+PHr7S1amM/SPPaYOQNKJD0Ubm7B3d1dv5RERO6jpCRYscLspVmyBJKTzfYiRcxtEp59Fh580NoaJXtSuBERkSwVG2s+RzN5Mhw5cr29USOzl6ZDB3M1YZGMsnTC3IYNG2jbti3BwcHYbDYWLlx42/OjoqJo3rw5RYoUwcfHhzp16rBixYqsKVZERDIsORm+/x46djSfmXnjDTPYFCwIL70EMTHmTKjOnRVs5N5ZGm4uXLhA1apV073b84YNG2jevDlLly5lx44dNG7cmLZt27Jz5877XKmIiGTEyZPw3/+aWyA0bw7z55sPDdeta64ofOwYjBsH5cpZXam4EqfZFdxms7FgwQI6dOhwV9dVrFiRzp07M3z48HSdfze7ioqIyN0zDNiwwVxsb/58c50aAB8fePppcxp35crW1ijZT47ZFTw5OZlz585RqFAhq0sREcnxzpwxe2M+/RR+++16e61a5rM0Tz0FefNaV5/kHNk63IwdO5YLFy7QqVOnW56TmJhIYmKi/TghISErShMRyREMA3780Qw0c+aYez6BGWK6djV7aWrWtLZGyXmybbiZPXs2I0eO5Ntvv8X/NrujRUZGMmrUqCysTETE9SUkwFdfmaFm9+7r7VWqmL003bqZw1AiVsiW4WbOnDn07t2buXPn0qxZs9ueO3ToUAYPHmw/TkhIoGjRove7RBERl7Rjh7kuzezZcOGC2eblZQ45Pf881K5tbpEgYqVsF25mz55Nr169mD17Nm3atLnj+Z6ennhqXqGISIZduGCGmU8/he3br7eXL28Gmh49zCndIs7C0nBz/vx5/vjjD/vxwYMHiY6OplChQhQrVoyhQ4dy7Ngxpk+fDpjBpkePHnzwwQc88sgjxMXFAZAnTx58fX0t+RpERFzV7t1moPnqK3MYCiB3bnOtmuefh/r11UsjzsnSqeDr1q2jcePGqdojIiKYNm0aPXv25NChQ6xbtw6ARo0asX79+luenx6aCi4icmuXLsHcuebQ0w8/XG8vVcoMND17QuHClpUnOdjd/P52mnVusorCjYhIar/9ZvbSfPklnD1rtnl4mFshPP88NGkCbpYu+yo5XY5Z50ZERDIuMREWLDB7aW7sFA8NheeeMzevDAy0rj6RjFK4ERHJgWbONPd0OnnSPHZzg8ceM6dxt2gB7u7W1idyLxRuRERykPPnoX9/c/gJ4IEHoE8f6N0btEqGuAqFGxGRHCI62lyPZt8+s6dm+HAYNsx8tkbElehHWkTExRkGTJgAQ4aYz9k88IA5LNWwodWVidwfCjciIi7szBlzyGnhQvP4scdg6lRN5xbXpol9IiIuavNmqFbNDDa5csH48bBokYKNuD6FGxERF5OUBG+/bQ47HT1qLsD3ww8wcKBWFJacQcNSIiIuJDYWnn4aVq82j7t1g4kTIX9+a+sSyUrquRERcRHLl0PVqmaw8fY2n62ZMUPBRnIehRsRkWzuyhV45RUICzMX5atSBXbsMPeB0jCU5EQalhIRycYOHIAuXWDrVvO4Xz/43//Ay8vaukSspHAjIpJNffMNPPssJCRAgQLwxRcQHm51VSLW07CUiEg2c/GiubFl585msKlb11x9WMFGxKRwIyKSjezdCw8/DJ99Zj5P8/rr5o7eoaFWVybiPDQsJSKSDRgGfP65uVbNpUsQEABffQXNmlldmYjzUbgREXFy8fHmMNQ335jHLVuau3oHBFhbl4iz0rCUiIgT27oVqlc3g42HB7z3HixdqmAjcjvquRERcULJyTB2rPlMzbVrULw4fP011K5tdWUizk/hRkTEyZw4ARER5orDAE8+CZMnm9O9ReTONCwlIuJEVq82t1BYvtxciO/TT2HOHAUbkbuhcCMi4gSuXYM33oDmzSEuDipUgG3bzAeJtYWCyN3RsJSIiMWOHIGuXWHzZvP42Wdh/Hhz80sRuXsKNyIiFlq4EHr1grNnwcfHfLamc2erqxLJ3jQsJSJigcuX4cUX4fHHzWDz0EOwc6eCjUhmULgREcli+/bBI4/Axx+bxy+/DJs2QcmS1tYl4io0LCUikoW+/BL69YMLF6BwYZg+HcLCrK5KxLUo3IiIZIFz5+CFF8z9oACaNIEZMyA42Nq6RFyRhqVERO6zn3+GGjXMYOPmBmPGwMqVCjYi94t6bkRE7hPDgI8+gn//G65cgZAQmD0bHn3U6spEXJvCjYjIfXD6NDzzDCxebB63bw9TpkChQtbWJZITaFhKRCSTbdhgbqGweDHkzm323ixYoGAjklUUbkREMklSEoweDY0bw7FjUKYM/PQT9O+vLRREspKGpUREMsGxY9C9O6xbZx5HRJjr2OTLZ2lZIjmSem5ERO7RkiVQrZoZbPLmNdeumTZNwUbEKgo3IiIZdOUKDBkCjz0Gp06ZAefnn+Hpp62uTCRn07CUiEgG/PEHdOkC27ebxwMGwHvvgaentXWJiMKNiMhdmz0bnn/eXHW4UCGYOhXatbO6KhFJoXAjIpJOFy6YPTRTppjHjz4Ks2ZB0aLW1iUijix95mbDhg20bduW4OBgbDYbCxcuvOM169evp2bNmnh5eVGyZEkmTZp0/wsVkRxvzx546CEz2NhsMHw4rF2rYCPijCwNNxcuXKBq1ap8/PHH6Tr/4MGDtG7dmvr167Nz505ef/11BgwYwPz58+9zpSKSUxkGTJpkBpuYGAgKgtWrYdQo8FDft4hTsvQ/zbCwMMLCwtJ9/qRJkyhWrBjjx48HoHz58mzfvp3//e9/PPHEE/epShHJqc6ehWefhZT/f2rd2pziXaSIpWWJyB1kq6ngP/zwAy1atHBoa9myJdu3b+fq1atpXpOYmEhCQoLDS0TkTn74AapXN4NNrlwwdqy5nYKCjYjzy1bhJi4ujoCAAIe2gIAArl27xqlTp9K8JjIyEl9fX/urqAbIReQ2kpPhP/+B+vXh8GEoWRI2b4bBg8EtW/2NKZJzZbv/VG03bdBiGEaa7SmGDh1KfHy8/XX06NH7XqOIZE9xcdCqFQwdau4T9dRTsHOn+byNiGQf2epxuMDAQOLi4hzaTpw4gYeHB35+fmle4+npiadW1RKRO1i50lxZ+MQJyJPH3BfqmWe04aVIdpStem7q1KnDqlWrHNpWrlxJrVq1yJUrl0VViUh2dvWq2VPTsqUZbCpVMlcd7tVLwUYku7I03Jw/f57o6Giio6MBc6p3dHQ0R44cAcwhpR49etjP79u3L4cPH2bw4MHExMQwZcoUvvjiC15++WUryheRbO7QIWjQwHzGBqBvX9i6FSpUsLQsEblHlg5Lbd++ncaNG9uPBw8eDEBERATTpk0jNjbWHnQASpQowdKlS3nppZf45JNPCA4O5sMPP9Q0cBG5a/PnQ+/eEB8Pvr7w+efQsaPVVYlIZrAZKU/k5hAJCQn4+voSHx+Pj4+P1eWISBa7dMmc+ZSyuPkjj5h7RRUvbmlZInIHd/P7O1s9cyMici9iYqB27evB5rXXYMMGBRsRV5OtZkuJiGSEYZg7d/fvb/bc+PvDjBlw05qgIuIiFG5ExKUlJJgPCs+ebR43a2YGm8BAa+sSkftHw1Ii4rK2b4caNcxg4+4OkZGwYoWCjYirU8+NiLic5GQYP958pubqVQgNNQNOnTpWVyYiWUHhRkRcysmT0LMnLF1qHoeHm9O8Cxa0tCwRyUIalhIRl7FlC1SrZgYbT0+YMAHmzVOwEclp1HMjIi5h6VJzEb5Ll6BcOZgzB6pUsboqEbGCem5EJNubPRvatzeDTZs25oPECjYiOZfCjYhkaxMmQLducO0adO0KCxZA3rxWVyUiVlK4EZFsyTBgzBjo18/89/79zfVrcuWyujIRsZrCjYhkO8nJ5v5Qb75pHg8fDh9+CG76G01E0APFIpLNXLsGffrAl1+ax+PHw8CBlpYkIk5G4UZEso3Ll+Gpp+Dbb80Vh6dMgR49rK5KRJyNwo2IZAsJCdChA6xda65h88030K6d1VWJiDNSuBERp3fyJISFwY4dkD8/LFoEjRpZXZWIOCuFGxFxakePQosW8NtvULgwLF8ONWtaXZWIODOFGxFxWvv2QfPmZsApWhRWrjRXHxYRuR1NnBQRp/Tzz1C/vhlsypaFTZsUbEQkfRRuRMTprF9vPlNz8iTUqAEbN0KxYlZXJSLZhcKNiDiVRYugZUs4dw4aNjRnRxUpYnVVIpKdKNyIiNOYMQPCwyEx0ZzmvWwZ+PhYXZWIZDcKNyLiFD74wFyQLynJ/Of8+ZAnj9VViUh2pHAjIpYyDBgxAgYNMo8HDoSpU8FDczlFJIP014eIWCY52QwzH39sHo8eDW+8ATabtXWJSPamcCMilrh6FZ55BmbONI8//hj69bO2JhFxDQo3IpLlLl2CJ5+EJUvM4acvv4SuXa2uSkRchcKNiGSp+Hho29Zcu8bLC+bNgzZtrK5KRFyJwo2IZJm//4ZWrSA62pzi/d135irEIiKZSeFGRLLE4cPmPlH794O/v7kBZvXqVlclIq5I4UZE7rtffzV39j52DEJDYdUqKF3a6qpExFVpnRsRua+2bYMGDcxgU6GCuQGmgo2I3E8KNyJy36xZA02awOnT8PDDsGEDhIRYXZWIuDqFGxG5LxYuhLAwOH8emjaF778HPz+rqxKRnEDhRkQy3dSp8MQTcOWKuRHmkiWQP7/VVYlITqFwIyKZatw46NXL3FqhVy+YMwc8Pa2uSkRyEoUbEckUhmHuCzVkiHn88svw+efaAFNEsp7+2hGRe5aUBP37w6RJ5nFkJLz6qjbAFBFrWN5zM2HCBEqUKIGXlxc1a9Zk48aNtz1/5syZVK1aFW9vb4KCgnjmmWc4ffp0FlUrIje7cgW6dTODjc1m/vO11xRsRMQ6loabOXPmMGjQIIYNG8bOnTupX78+YWFhHDlyJM3zN23aRI8ePejduzd79+5l7ty5bNu2jT59+mRx5SICcPEitG9vPleTKxfMng3PP291VSKS01kabsaNG0fv3r3p06cP5cuXZ/z48RQtWpSJEyemef6PP/5I8eLFGTBgACVKlODRRx/l+eefZ/v27VlcuYicPWtup7B8OXh7w+LF0Lmz1VWJiFgYbq5cucKOHTto0aKFQ3uLFi3YsmVLmtfUrVuXv/76i6VLl2IYBn///Tfz5s2jzW22FE5MTCQhIcHhJSL3Ji4OGjWCLVugQAFzO4WWLa2uSkTEZFm4OXXqFElJSQQEBDi0BwQEEBcXl+Y1devWZebMmXTu3JncuXMTGBhIgQIF+Oijj255n8jISHx9fe2vokWLZurXIZLTHDwIjz4Ku3dDYCCsXw9161pdlYjIdZY/UGy76alDwzBStaX49ddfGTBgAMOHD2fHjh0sX76cgwcP0rdv31t+/tChQ4mPj7e/jh49mqn1i+Qkv/wC9erBn39CiRLmPlFVqlhdlYiII8umghcuXBh3d/dUvTQnTpxI1ZuTIjIyknr16vHvf/8bgCpVqpA3b17q16/PmDFjCAoKSnWNp6cnnlpBTOSe/fgjtG5tPmtTqRKsWAHBwVZXJSKSmmU9N7lz56ZmzZqsWrXKoX3VqlXUvUUf98WLF3FzcyzZ3d0dMHt8ROT+WLXK3B/q7Fl45BFzKErBRkSclaXDUoMHD+bzzz9nypQpxMTE8NJLL3HkyBH7MNPQoUPp0aOH/fy2bdsSFRXFxIkTOXDgAJs3b2bAgAE8/PDDBOtvWpH7Yt48aNPGnPbdooW5AWahQlZXJSJya5auUNy5c2dOnz7N6NGjiY2NpVKlSixdupTQ0FAAYmNjHda86dmzJ+fOnePjjz9myJAhFChQgCZNmvDuu+9a9SWIuLTPPoO+fc19op58EmbM0D5RIuL8bEYOG89JSEjA19eX+Ph4fHx8rC5HxGm9+6650jDAs8/CxInw/6PAIiJZ7m5+f1s+W0pEnIthmPtCpQSb116DTz9VsBGR7EMbZ4qIXVKSOQz1+efm8Xvvwf9PThQRyTYUbkQEgMRE6N7dfIDYzc3srdG2bSKSHSnciAjnz0N4uDnlO3dumDULnnjC6qpERDJG4UYkhztzxlyc76efIG9eWLgQmjWzuioRkYxTuBHJwY4fN9eu2bsXChaEZcugdm2rqxIRuTcZmi119OhR/vrrL/vx1q1bGTRoEJMnT860wkTk/vrjD3OfqL17zdWGN25UsBER15ChcNO1a1fWrl0LQFxcHM2bN2fr1q28/vrrjB49OlMLFJHMt3u3ubP3oUNQqhRs3gwVK1pdlYhI5shQuPnll194+OGHAfjmm2+oVKkSW7ZsYdasWUybNi0z6xORTLZ5MzRoAH//DVWrmjt7Fy9udVUiIpknQ+Hm6tWr9p22v//+e9q1awdAuXLliI2NzbzqRCRTLV8OzZtDfLw5JLVuHQQEWF2ViEjmylC4qVixIpMmTWLjxo2sWrWKVq1aAXD8+HH8/PwytUARyRxffw1t28KlSxAWBitXQoECVlclIpL5MhRu3n33XT799FMaNWpEly5dqFq1KgCLFi2yD1eJiPOYNAm6doVr16BLF3O6t7e31VWJiNwfGd44MykpiYSEBAoWLGhvO3ToEN7e3vj7+2dagZlNG2dKTmIYEBkJw4aZxy+8AB99ZK5ALCKSndz3jTMvXbpEYmKiPdgcPnyY8ePHs2/fPqcONiI5iWHAyy9fDzZvvAEff6xgIyKuL0N/zbVv357p06cD8M8//1C7dm3Gjh1Lhw4dmDhxYqYWKCJ379o16N0bxo0zj99/H956C2w2a+sSEckKGQo3P//8M/Xr1wdg3rx5BAQEcPjwYaZPn86HH36YqQWKyN25fBmefBKmTgV3d5g2DQYNsroqEZGsk6HtFy5evEj+/PkBWLlyJeHh4bi5ufHII49w+PDhTC1QRNLv3Dno0AHWrAFPT5gzB9q3t7oqEZGslaGem1KlSrFw4UKOHj3KihUraNGiBQAnTpzQQ7oiFjl1Cpo0MYNNvnzmPlEKNiKSE2Uo3AwfPpyXX36Z4sWL8/DDD1OnTh3A7MWpXr16phYoInd29CjUrw/bt4OfH6xdC40bW12ViIg1MjwVPC4ujtjYWKpWrYrb/0+/2Lp1Kz4+PpQrVy5Ti8xMmgourub3381Vh48cgZAQc3G+8uWtrkpEJHPdze/vDD1zAxAYGEhgYCB//fUXNpuNBx54QAv4iWSxn3+GVq3g5EkoUwZWrYJixayuSkTEWhkalkpOTmb06NH4+voSGhpKsWLFKFCgAG+99RbJycmZXaOIpGHDBnPo6eRJqF4dNm5UsBERgQz23AwbNowvvviC//znP9SrVw/DMNi8eTMjR47k8uXLvP3225ldp9NLSjJ/ucTGQlCQ+fyDu7vVVYmr+u47c7r35cvmDt+LFoGvr9VViYg4hww9cxMcHMykSZPsu4Gn+Pbbb3nhhRc4duxYphWY2e7HMzdRUTBwIPz11/W2kBD44AMID8+UW4jYzZwJERFmoH7sMfjmG8iTx+qqRETur/u+/cKZM2fSfGi4XLlynDlzJiMfmW1FRUHHjo7BBuDYMbM9KsqausQ1ffQRdO9uBpvu3c2fLwUbERFHGQo3VatW5eOPP07V/vHHH1OlSpV7Liq7SEoye2zS6vtKaRs0yDxP5F4YBowaBQMGmMcvvghffgm5cllbl4iIM8rQMzfvvfcebdq04fvvv6dOnTrYbDa2bNnC0aNHWbp0aWbX6LQ2bkzdY3MjwzDXH9m4ERo1yrKyxMUkJ8NLL0HKziYjR8Lw4donSkTkVjLUc9OwYUN+//13Hn/8cf755x/OnDlDeHg4e/fuZerUqZldo9OKjc3c80RudPo0fPopPPro9WDz4YcwYoSCjYjI7WR4Eb+07Nq1ixo1apDkxOMwmflA8bp16VsFdu1a9dxI+ly4YM58mjULli83d/cGc/hpyhTzORsRkZwoSxbxE3O6d0iI+fBwWhHRZjPf//8N1EXSdOWKuarwrFnw7bdw8eL196pXh65d4amnzJ8lERG5M4Wbe+Dubk737tjRDDI3BpyUYYPx47XejaSWnAybNpmBZu5cuHGS4YMPmoGmSxdtoyAikhEKN/coPBzmzUt7nZvx47XOjVxnGLBrlxloZs92/HkJCDB7Z7p2hYce0jM1IiL34q7CTfgdflP/888/91JLthUeDu3ba4ViSduff5phZtYsiIm53u7jA088YQaaxo318yIiklnuKtz43mF9d19fX3r06HFPBWVX7u56aFiui4szVw6eNQt++ul6u6cntG1rBpqwMPDysq5GERFXdVfhJidN8xa5W/HxsGCBGWhWrzafqwFwc4NmzcxA06GD9oASEbnf9MyNyD24fBmWLjUDzXffQWLi9fceecQMNJ06mc/UiIhI1lC4EblLSUnm2kWzZsH8+ZCQcP298uWhWzfz4eAHH7SuRhGRnEzhRiQdDAO2bjUDzZw58Pff198rWtSctt21K1SpoplOIiJWU7gRuY2YGDPQzJoFBw5cby9UyBxu6toV6tUzn6sRERHnYPlfyRMmTKBEiRJ4eXlRs2ZNNm7ceNvzExMTGTZsGKGhoXh6evLggw8yZcqULKr21gwDhgwx9wL66SdzGX3Jno4ehf/+11wduEIFGDPGDDZ585ph5rvvzCn/EyeaU/4VbEREnIulPTdz5sxh0KBBTJgwgXr16vHpp58SFhbGr7/+SrFixdK8plOnTvz999988cUXlCpVihMnTnAtZQMeCx0/DuPGXT+22aBMGahaFapVM/9ZtSoEB2vYwhmdPm0uxjhrFmzYcL3dwwNatTKfo2nb1gw4IiLi3DJ148y7Vbt2bWrUqMHEiRPtbeXLl6dDhw5ERkamOn/58uU89dRTHDhwgEKFCmXonpm5ceaNYmPNrRiio81VaOPi0j6vcGHHwFOtGpQrZ26MKFnrVptUAjRsaPbSPPEE+PlZV6OIiJju5ve3ZeHmypUreHt7M3fuXB5//HF7+8CBA4mOjmb9+vWprnnhhRf4/fffqVWrFjNmzCBv3ry0a9eOt956izx58qR5n8TERBJvmJ+bkJBA0aJFMz3c3Ozvv82QkxJ2oqNh3z5zps3NcueGihVT9/IULHjfysuxrl69vknlwoVpb1LZubP5kLCIiDiPbLEr+KlTp0hKSiLgpgVAAgICiLtFt8eBAwfYtGkTXl5eLFiwgFOnTvHCCy9w5syZWz53ExkZyahRozK9/jsJCIAWLcxXikuXYO9eM+zcGHwSEmDnTvN1o2LFHHt4qlaFEiX0jMfdSk6GzZuvb1J5+vT197RJpYiI67F8tpTtpgdQDMNI1ZYiOTkZm83GzJkz7VtBjBs3jo4dO/LJJ5+k2XszdOhQBg8ebD9O6bmxQp48UKuW+UphGHDoUOpenkOH4MgR87Vo0fXz8+c3pxvfGHoqVTI/W64zDNi9+/omlUePXn9Pm1SKiLg2y8JN4cKFcXd3T9VLc+LEiVS9OSmCgoJ44IEHHPa4Kl++PIZh8Ndff1G6dOlU13h6euLp6Zm5xWcim83sjSlRwlyaP8U//5i/nG8MPb/8AufOmb0QmzdfP9fNDcqWTf0sT2Bgln4pTuHAgeubVP766/V2bVIpIpJzWBZucufOTc2aNVm1apXDMzerVq2iffv2aV5Tr1495s6dy/nz58mXLx8Av//+O25uboSEhGRJ3VmlQAFo0MB8pbh2zXxu58YenuhoOHnSXI8lJga+/vr6+f7+qQNP2bLmDCBX8vff5sJ6aW1S+dhjZqBp3VqbVIqI5BSWzpaaM2cOTz/9NJMmTaJOnTpMnjyZzz77jL179xIaGsrQoUM5duwY06dPB+D8+fOUL1+eRx55hFGjRnHq1Cn69OlDw4YN+eyzz9J1z/s1W8oqhmHOzLp5WOv3369v3HgjT09zGOvmh5ez22aOt9uksmlTM9A8/nj2+7pERCRt2eKBYoDOnTtz+vRpRo8eTWxsLJUqVWLp0qWEhoYCEBsby5EjR+zn58uXj1WrVvHiiy9Sq1Yt/Pz86NSpE2PGjLHqS7CczQZBQearVavr7RcvmsNYN4aeXbvg/HnYscN83ah48dQPLxcv7lzPo6Rnk8onn8yZw3EiInKdpT03VnC1npu7kZwMBw+m7uW5IT868PVN/fByxYpZO7yjTSpFRASyyTo3VsnJ4eZWzpxJ/fDy3r1w5Urqc93dzUUHbx7WusUz4BliGLBt2/VNKm985lybVIqI5EwKN7ehcJM+V6/Cb7859vDs2gWnTqV9fmBg6mGtMmXublZSTMz1mU5//nm9XZtUioiIws1tKNxknGGYe2jdPKy1f7/53s28vKByZcfAU6WKOS07xV9/mTO8Zs1yXMTQ29ucGt+1KzRvbq7iLCIiOZfCzW0o3GS+Cxdgzx7H0LN79613Ri9Z0gw7p0+bm1Sm/ASmbFLZtSu0a6dNKkVE5DqFm9tQuMkaycnm0NLNw1p//ZX63AYNzEDTsaM2qRQRkbRlm6ng4rrc3KB0afP15JPX20+fvj4t3c0NwsO1SaWIiGQuhRvJUn5+0KSJ+RIREbkfNO9EREREXIrCjYiIiLgUhRsRERFxKQo3IiIi4lIUbkRERMSlKNyIiIiIS1G4EREREZeicCMiIiIuRYv4SZZLSoKNGyE2FoKCoH79u9s9XERE5HYUbiRLRUXBwIGOe0yFhMAHH5hbMYiIiNwrDUtJlomKMjfHvHnzzGPHzPaoKGvqEhER16JwI1kiKcnssUlrD/qUtkGDzPNERETuhcKNZImNG1P32NzIMODoUfM8ERGRe6FwI1kiNjZzzxMREbkVhRvJEkFBmXueiIjIrSjcSJaoX9+cFWWzpf2+zQZFi5rniYiI3AuFG8kS7u7mdG9IHXBSjseP13o3IiJy7xRuJMuEh8O8efDAA47tISFmu9a5ERGRzKBF/CRLhYdD+/ZaoVhERO4fhRvJcu7u0KiR1VWIiIir0rCUiIiIuBSFGxEREXEpCjciIiLiUhRuRERExKUo3IiIiIhLUbgRERERl6JwIyIiIi5F4UZERERcisKNiIiIuBSFGxEREXEpCjciIiLiUiwPNxMmTKBEiRJ4eXlRs2ZNNm7cmK7rNm/ejIeHB9WqVbu/BYqIiEi2Ymm4mTNnDoMGDWLYsGHs3LmT+vXrExYWxpEjR257XXx8PD169KBp06ZZVKmIiIhkFzbDMAyrbl67dm1q1KjBxIkT7W3ly5enQ4cOREZG3vK6p556itKlS+Pu7s7ChQuJjo5O9z0TEhLw9fUlPj4eHx+feylfREREssjd/P62rOfmypUr7NixgxYtWji0t2jRgi1bttzyuqlTp/Lnn38yYsSIdN0nMTGRhIQEh5eIiIi4LsvCzalTp0hKSiIgIMChPSAggLi4uDSv2b9/P6+99hozZ87Ew8MjXfeJjIzE19fX/ipatOg91y4iIiLOy/IHim02m8OxYRip2gCSkpLo2rUro0aNokyZMun+/KFDhxIfH29/HT169J5rFhEREeeVvu6P+6Bw4cK4u7un6qU5ceJEqt4cgHPnzrF9+3Z27txJ//79AUhOTsYwDDw8PFi5ciVNmjRJdZ2npyeenp7354sQERERp2NZz03u3LmpWbMmq1atcmhftWoVdevWTXW+j48Pe/bsITo62v7q27cvZcuWJTo6mtq1a2dV6SIiIuLELOu5ARg8eDBPP/00tWrVok6dOkyePJkjR47Qt29fwBxSOnbsGNOnT8fNzY1KlSo5XO/v74+Xl1eqdhEREcm5LA03nTt35vTp04wePZrY2FgqVarE0qVLCQ0NBSA2NvaOa96IiIiI3MjSdW6soHVuREREsp9ssc6NiIiIyP2gcCMiIiIuReFGREREXIrCjYiIiLgUhRsRERFxKQo3IiIi4lIUbkRERMSlKNyIiIiIS1G4EREREZeicCMiIiIuReFGREREXIrCjYiIiLgUhRsRERFxKQo3IiIi4lIUbkRERMSlKNyIiIiIS/GwugARubWkJNi4EWJjISgI6tcHd3erqxIRcW4KNyJOKioKBg6Ev/663hYSAh98AOHh1tUlIuLsNCwl4oSioqBjR8dgA3DsmNkeFWVNXSIi2YHCjYiTSUoye2wMI/V7KW2DBpnniYhIago3Ik5m48bUPTY3Mgw4etQ8T0REUlO4EXEysbGZe56ISE6jcCPiZIKCMvc8EZGcRuFGxMnUr2/OirLZ0n7fZoOiRc3zREQkNYUbESfj7m5O94bUASflePx4rXcjInIrCjciTig8HObNgwcecGwPCTHbtc6NiMitaRE/EScVHg7t22uFYhGRu6VwI+LE3N2hUSOrqxARyV40LCUiIiIuReFGREREXIrCjYiIiLgUhRsRERFxKQo3IiIi4lIUbkRERMSlKNyIiIiIS1G4EREREZeicCMiIiIuReFGREREXIrl4WbChAmUKFECLy8vatasycaNG295blRUFM2bN6dIkSL4+PhQp04dVqxYkYXVioiIiLOzNNzMmTOHQYMGMWzYMHbu3En9+vUJCwvjyJEjaZ6/YcMGmjdvztKlS9mxYweNGzembdu27Ny5M4srFxEREWdlMwzDsOrmtWvXpkaNGkycONHeVr58eTp06EBkZGS6PqNixYp07tyZ4cOHp+v8hIQEfH19iY+Px8fHJ0N1i4iISNa6m9/flvXcXLlyhR07dtCiRQuH9hYtWrBly5Z0fUZycjLnzp2jUKFC96NEERERyYY8rLrxqVOnSEpKIiAgwKE9ICCAuLi4dH3G2LFjuXDhAp06dbrlOYmJiSQmJtqPExISMlawiIiIZAuWP1Bss9kcjg3DSNWWltmzZzNy5EjmzJmDv7//Lc+LjIzE19fX/ipatOg91ywiIiLOy7JwU7hwYdzd3VP10pw4cSJVb87N5syZQ+/evfnmm29o1qzZbc8dOnQo8fHx9tfRo0fvuXYRERFxXpaFm9y5c1OzZk1WrVrl0L5q1Srq1q17y+tmz55Nz549mTVrFm3atLnjfTw9PfHx8XF4iYiIiOuy7JkbgMGDB/P0009Tq1Yt6tSpw+TJkzly5Ah9+/YFzF6XY8eOMX36dMAMNj169OCDDz7gkUcesff65MmTB19fX8u+DhEREXEeloabzp07c/r0aUaPHk1sbCyVKlVi6dKlhIaGAhAbG+uw5s2nn37KtWvX6NevH/369bO3R0REMG3atKwuX0RERJyQpevcWEHr3IiIiGQ/2WKdGxEREZH7QeFGREREXIrCjYiIiLgUhRsRERFxKQo3IiIi4lIUbkRERMSlKNyIiIiIS1G4EREREZeicCMiIiIuReFGREREXIrCjYiIiLgUhRsRERFxKQo3IiIi4lI8rC5ARCQzJCXBxo0QGwtBQVC/Pri7W12ViFhB4UZEsr2oKBg4EP7663pbSAh88AGEh1tXl4hYQ8NSIpKtRUVBx46OwQbg2DGzPSrKmrpExDoKNyKSbSUlmT02hpH6vZS2QYPM80Qk51C4EZFsa+PG1D02NzIMOHrUPE9Ecg6FGxHJtmJjM/c8EXENCjcikm0FBWXueSLiGhRuRCTbql/fnBVls6X9vs0GRYua54lIzqFwIyLZlru7Od0bUgeclOPx47XejUhOo3AjItlaeDjMmwcPPODYHhJitmudG5GcR4v4iUi2Fx4O7dtrhWIRMSnciIhLcHeHRo2srkJEnIGGpURERMSlKNyIiIiIS1G4EREREZeiZ25ERHKgpCQ9gC2uS+FGRCSHiYoyNxy9cV+ukBBzzSBNnRdXoGEpEZEcJCoKOnZMveHosWNme1SUNXWJa0hKgnXrYPZs859JSdbUoXAjIpJDJCWZPTaGkfq9lLZBg6z7hSTZW1QUFC8OjRtD167mP4sXtyYwK9yIiOQQGzem7rG5kWHA0aPmeSJ3w9l6BBVuRERyiNjYzD0vp3CWoRZn5Yw9ggo3IiI5RFBQ5p6XEzjTUIuzcsYeQYUbEZEcon59c1bUzTuop7DZoGhR8zxxvqEWZ+WMPYIKNyIiOYS7uzndG1IHnJTj8eO13g0451CLs3LGHkGFGxGRHCQ8HObNgwcecGwPCTHbtc6NyRmHWpyVM/YIahE/EZEcJjwc2rfXCsW344xDLc4qpUewY0czyNzY22VVj6DlPTcTJkygRIkSeHl5UbNmTTbeIQavX7+emjVr4uXlRcmSJZk0aVIWVSoi4jrc3aFRI+jSxfyngo0jZxxqcWbO1iNoabiZM2cOgwYNYtiwYezcuZP69esTFhbGkSNH0jz/4MGDtG7dmvr167Nz505ef/11BgwYwPz587O4chERcWXOONTi7MLD4dAhWLsWZs0y/3nwoDVDnTbDSOtxqaxRu3ZtatSowcSJE+1t5cuXp0OHDkRGRqY6/9VXX2XRokXExMTY2/r27cuuXbv44Ycf0nXPhIQEfH19iY+Px8fH596/CBERcUkps6Ug7aEWPaOUte7m97dlPTdXrlxhx44dtGjRwqG9RYsWbNmyJc1rfvjhh1Tnt2zZku3bt3P16tU0r0lMTCQhIcHhJSIicifONtQi6WdZuDl16hRJSUkEBAQ4tAcEBBAXF5fmNXFxcWmef+3aNU6dOpXmNZGRkfj6+tpfRYsWzZwvQEREXJ4zDbVI+lk+W8p204CmYRip2u50flrtKYYOHcrgwYPtxwkJCQo4IiKSbikPX0v2YVm4KVy4MO7u7ql6aU6cOJGqdyZFYGBgmud7eHjg5+eX5jWenp54enpmTtEiIiLi9CwblsqdOzc1a9Zk1apVDu2rVq2ibt26aV5Tp06dVOevXLmSWrVqkStXrvtWq4iIiGQflk4FHzx4MJ9//jlTpkwhJiaGl156iSNHjtC3b1/AHFLq0aOH/fy+ffty+PBhBg8eTExMDFOmTOGLL77g5ZdftupLEBERESdj6TM3nTt35vTp04wePZrY2FgqVarE0qVLCQ0NBSA2NtZhzZsSJUqwdOlSXnrpJT755BOCg4P58MMPeeKJJ6z6EkRERMTJWLrOjRW0zo2IiEj2ky3WuRERERG5HxRuRERExKUo3IiIiIhLUbgRERERl2L5CsVZLeX5ae0xJSIikn2k/N5OzzyoHBduzp07B6AtGERERLKhc+fO4evre9tzctxU8OTkZI4fP07+/Plvu4dVRqTsW3X06FFNM78Dfa/ST9+r9NP36u7o+5V++l6l3/36XhmGwblz5wgODsbN7fZP1eS4nhs3NzdCQkLu6z18fHz0w59O+l6ln75X6afv1d3R9yv99L1Kv/vxvbpTj00KPVAsIiIiLkXhRkRERFyKwk0m8vT0ZMSIEXh6elpditPT9yr99L1KP32v7o6+X+mn71X6OcP3Ksc9UCwiIiKuTT03IiIi4lIUbkRERMSlKNyIiIiIS1G4EREREZeicJMJNmzYQNu2bQkODsZms7Fw4UKrS3JakZGRPPTQQ+TPnx9/f386dOjAvn37rC7LKU2cOJEqVarYF8KqU6cOy5Yts7qsbCEyMhKbzcagQYOsLsXpjBw5EpvN5vAKDAy0uiyndezYMbp3746fnx/e3t5Uq1aNHTt2WF2WUypevHiqny2bzUa/fv2yvBaFm0xw4cIFqlatyscff2x1KU5v/fr19OvXjx9//JFVq1Zx7do1WrRowYULF6wuzemEhITwn//8h+3bt7N9+3aaNGlC+/bt2bt3r9WlObVt27YxefJkqlSpYnUpTqtixYrExsbaX3v27LG6JKd09uxZ6tWrR65cuVi2bBm//vorY8eOpUCBAlaX5pS2bdvm8HO1atUqAJ588sksryXHbb9wP4SFhREWFmZ1GdnC8uXLHY6nTp2Kv78/O3bsoEGDBhZV5Zzatm3rcPz2228zceJEfvzxRypWrGhRVc7t/PnzdOvWjc8++4wxY8ZYXY7T8vDwUG9NOrz77rsULVqUqVOn2tuKFy9uXUFOrkiRIg7H//nPf3jwwQdp2LBhlteinhuxVHx8PACFChWyuBLnlpSUxNdff82FCxeoU6eO1eU4rX79+tGmTRuaNWtmdSlObf/+/QQHB1OiRAmeeuopDhw4YHVJTmnRokXUqlWLJ598En9/f6pXr85nn31mdVnZwpUrV/jqq6/o1atXpm9SnR4KN2IZwzAYPHgwjz76KJUqVbK6HKe0Z88e8uXLh6enJ3379mXBggVUqFDB6rKc0tdff82OHTuIjIy0uhSnVrt2baZPn86KFSv47LPPiIuLo27dupw+fdrq0pzOgQMHmDhxIqVLl2bFihX07duXAQMGMH36dKtLc3oLFy7kn3/+oWfPnpbcX8NSYpn+/fuze/duNm3aZHUpTqts2bJER0fzzz//MH/+fCIiIli/fr0Czk2OHj3KwIEDWblyJV5eXlaX49RuHEKvXLkyderU4cEHH+TLL79k8ODBFlbmfJKTk6lVqxbvvPMOANWrV2fv3r1MnDiRHj16WFydc/viiy8ICwsjODjYkvur50Ys8eKLL7Jo0SLWrl1LSEiI1eU4rdy5c1OqVClq1apFZGQkVatW5YMPPrC6LKezY8cOTpw4Qc2aNfHw8MDDw4P169fz4Ycf4uHhQVJSktUlOq28efNSuXJl9u/fb3UpTicoKCjV/0iUL1+eI0eOWFRR9nD48GG+//57+vTpY1kN6rmRLGUYBi+++CILFixg3bp1lChRwuqSshXDMEhMTLS6DKfTtGnTVDN+nnnmGcqVK8err76Ku7u7RZU5v8TERGJiYqhfv77VpTidevXqpVqq4vfffyc0NNSiirKHlIkibdq0sawGhZtMcP78ef744w/78cGDB4mOjqZQoUIUK1bMwsqcT79+/Zg1axbffvst+fPnJy4uDgBfX1/y5MljcXXO5fXXXycsLIyiRYty7tw5vv76a9atW5dqxplA/vz5Uz23lTdvXvz8/PQ8101efvll2rZtS7FixThx4gRjxowhISGBiIgIq0tzOi+99BJ169blnXfeoVOnTmzdupXJkyczefJkq0tzWsnJyUydOpWIiAg8PCyMGIbcs7Vr1xpAqldERITVpTmdtL5PgDF16lSrS3M6vXr1MkJDQ43cuXMbRYoUMZo2bWqsXLnS6rKyjYYNGxoDBw60ugyn07lzZyMoKMjIlSuXERwcbISHhxt79+61uiyntXjxYqNSpUqGp6enUa5cOWPy5MlWl+TUVqxYYQDGvn37LK3DZhiGYU2sEhEREcl8eqBYREREXIrCjYiIiLgUhRsRERFxKQo3IiIi4lIUbkRERMSlKNyIiIiIS1G4EREREZeicCMiOZLNZmPhwoVWlyEi94HCjYhkuZ49e2Kz2VK9WrVqZXVpIuICtLeUiFiiVatWTJ061aHN09PTompExJWo50ZELOHp6UlgYKDDq2DBgoA5ZDRx4kTCwsLIkycPJUqUYO7cuQ7X79mzhyZNmpAnTx78/Px47rnnOH/+vMM5U6ZMoWLFinh6ehIUFET//v0d3j916hSPP/443t7elC5dmkWLFtnfO3v2LN26daNIkSLkyZOH0qVLpwpjIuKcFG5ExCm9+eabPPHEE+zatYvu3bvTpUsXYmJiALh48SKtWrWiYMGCbNu2jblz5/L99987hJeJEyfSr18/nnvuOfbs2cOiRYsoVaqUwz1GjRpFp06d2L17N61bt6Zbt26cOXPGfv9ff/2VZcuWERMTw8SJEylcuHDWfQNEJOMs3bZTRHKkiIgIw93d3cibN6/Da/To0YZhmLvH9+3b1+Ga2rVrG//6178MwzCMyZMnGwULFjTOnz9vf3/JkiWGm5ubERcXZxiGYQQHBxvDhg27ZQ2A8cYbb9iPz58/b9hsNmPZsmWGYRhG27ZtjWeeeSZzvmARyVJ65kZELNG4cWMmTpzo0FaoUCH7v9epU8fhvTp16hAdHQ1ATEwMVatWJW/evPb369WrR3JyMvv27cNms3H8+HGaNm162xqqVKli//e8efOSP39+Tpw4AcC//vUvnnjiCX7++WdatGhBhw4dqFu3boa+VhHJWgo3ImKJvHnzphomuhObzQaAYRj2f0/rnDx58qTr83LlypXq2uTkZADCwsI4fPgwS5Ys4fvvv6dp06b069eP//3vf3dVs4hkPT1zIyJO6ccff0x1XK5cOQAqVKhAdHQ0Fy5csL+/efNm3NzcKFOmDPnz56d48eKsXr36nmooUqQIPXv25KuvvmL8+PFMnjz5nj5PRLKGem5ExBKJiYnExcU5tHl4eNgf2p07dy61atXi0UcfZebMmWzdupUvvvgCgG7dujFixAgiIiIYOXIkJ0+e5MUXX+Tpp58mICAAgJEjR9K3b1/8/f0JCwvj3LlzbN68mRdffDFd9Q0fPpyaNWtSsWJFEhMT+e677yhfvnwmfgdE5H5RuBERSyxfvpygoCCHtrJly/Lbb78B5kymr7/+mhdeeIHAwEBmzpxJhQoVAPD29mbFihUMHDiQhx56CG9vb5544gnGjRtn/6yIiAguX77M+++/z8svv0zhwoXp2LFjuuvLnTs3Q4cO5dChQ+TJk4f69evz9ddfZ8JXLiL3m80wDMPqIkREbmSz2ViwYAEdOnSwuhQRyYb0zI2IiIi4FIUbERERcSl65kZEnI5Gy0XkXqjnRkRERFyKwo2IiIi4FIUbERERcSkKNyIiIuJSFG5ERETEpSjciIiIiEtRuBERERGXonAjIiIiLkXhRkRERFzK/wGudKp7jEQUTAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "acc = train_state['train_acc']\n",
    "val_acc = train_state['val_acc']\n",
    "loss = train_state['train_loss']\n",
    "val_loss = train_state['val_loss']\n",
    "\n",
    "epochs = range(1, len(acc) + 1)\n",
    "\n",
    "# \"bo\" is for \"blue dot\"\n",
    "plt.plot(epochs, loss, 'bo', label='Training loss')\n",
    "# b is for \"solid blue line\"\n",
    "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n",
    "plt.title('Training and validation loss')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Loss')\n",
    "plt.legend()\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAHFCAYAAAAUpjivAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAA9hAAAPYQGoP6dpAABZnklEQVR4nO3deXhMZ/8G8HuyTfYgskwkJrHvuyJKLLVTe62V2EqtaW3VFlFEaS1ttfHyamwl2grdXrVTigoataZoSJA01iyWkUye3x/nl5ExWSZMMifj/lzXuThnnnPmO2Ni7jzPc85RCCEEiIiIiCyUlbkLICIiIipODDtERERk0Rh2iIiIyKIx7BAREZFFY9ghIiIii8awQ0RERBaNYYeIiIgsGsMOERERWTSGHSIiIrJoDDtkMRQKhVHLgQMHXuh5wsLCoFAonmvfAwcOmKQGuQsJCYG/v78sntff3x8hISGF7vsi/zZHjhxBWFgY7t+/b/BYmzZt0KZNmyIfk4hMx8bcBRCZytGjR/XW582bh/3792Pfvn1622vVqvVCzzNq1Ch07tz5ufZt1KgRjh49+sI1kPG2bdsGV1fXYn2OI0eOYO7cuQgJCUGZMmX0Hvvqq6+K9bmJqHAMO2Qxmjdvrrfu4eEBKysrg+3PevjwIRwdHY1+Hl9fX/j6+j5Xja6uroXWQ6bVsGFDsz4/g61xMjMzoVAoYGPDryUyPQ5j0UulTZs2qFOnDn777TcEBgbC0dERI0aMAABs2bIFHTt2hEqlgoODA2rWrIn33nsPDx480DtGXsNY/v7+6N69O3799Vc0atQIDg4OqFGjBr7++mu9dnkNlYSEhMDZ2RmXL19G165d4ezsDD8/P0yZMgUajUZv/+vXr6Nfv35wcXFBmTJlMGTIEMTExEChUGDt2rUFvvZbt25h3LhxqFWrFpydneHp6Yl27drh0KFDeu2uXr0KhUKBTz/9FEuXLkVAQACcnZ3RokULHDt2zOC4a9euRfXq1aFUKlGzZk2sX7++wDpy9OrVC2q1GtnZ2QaPNWvWDI0aNdKtf/nll2jdujU8PT3h5OSEunXrYvHixcjMzCz0efIaxrp48SI6d+4MR0dHlC9fHmPHjkV6errBvrt370bPnj3h6+sLe3t7VKlSBWPGjMHt27d1bcLCwjBt2jQAQEBAgMFwaV7DWHfv3sW4ceNQoUIF2NnZoVKlSvjggw8M/r0VCgUmTJiADRs2oGbNmnB0dET9+vXx888/F/q6Hz9+jClTpqBBgwZwc3NDuXLl0KJFC/zwww8GbbOzs/HFF1+gQYMGcHBwQJkyZdC8eXP8+OOPeu02bdqEFi1awNnZGc7OzmjQoAHWrFlT4Hud13uQ83OwYcMGTJkyBRUqVIBSqcTly5eN/pwCgEajwUcffYSaNWvC3t4e7u7uaNu2LY4cOQIAaN++PWrUqIFn73cthECVKlXQrVu3Qt9HsgyM0PTSSUpKwtChQzF9+nSEh4fDykrK/JcuXULXrl0RGhoKJycnXLx4EYsWLcLx48cNhsLycvr0aUyZMgXvvfcevLy88N///hcjR45ElSpV0Lp16wL3zczMxOuvv46RI0diypQp+O233zBv3jy4ublh9uzZAIAHDx6gbdu2uHv3LhYtWoQqVarg119/xYABA4x63Xfv3gUAzJkzB97e3sjIyMC2bdvQpk0b7N271+AL+csvv0SNGjWwfPlyAMCsWbPQtWtXxMfHw83NDYAUdIYPH46ePXtiyZIlSE1NRVhYGDQaje59zc+IESPQs2dP7Nu3D6+99ppu+8WLF3H8+HF8/vnnum1XrlzB4MGDERAQADs7O5w+fRoLFizAxYsXDQJlYf79918EBQXB1tYWX331Fby8vPDNN99gwoQJBm2vXLmCFi1aYNSoUXBzc8PVq1exdOlSvPrqqzhz5gxsbW0xatQo3L17F1988QWio6OhUqkA5N+j8/jxY7Rt2xZXrlzB3LlzUa9ePRw6dAgLFy5EbGwsfvnlF732v/zyC2JiYvDRRx/B2dkZixcvRu/evREXF4dKlSrl+zo1Gg3u3r2LqVOnokKFCnjy5An27NmDPn36IDIyEsOGDdO1DQkJwcaNGzFy5Eh89NFHsLOzw6lTp3D16lVdm9mzZ2PevHno06cPpkyZAjc3N5w9exbXrl0rytuvZ+bMmWjRogVWrlwJKysreHp64tatWwAK/5xmZWWhS5cuOHToEEJDQ9GuXTtkZWXh2LFjSEhIQGBgICZPnoyePXti7969ep+xHTt24MqVK3qfMbJwgshCBQcHCycnJ71tQUFBAoDYu3dvgftmZ2eLzMxMcfDgQQFAnD59WvfYnDlzxLM/Omq1Wtjb24tr167ptj169EiUK1dOjBkzRrdt//79AoDYv3+/Xp0AxLfffqt3zK5du4rq1avr1r/88ksBQOzYsUOv3ZgxYwQAERkZWeBrelZWVpbIzMwU7du3F71799Ztj4+PFwBE3bp1RVZWlm778ePHBQCxefNmIYQQWq1W+Pj4iEaNGons7Gxdu6tXrwpbW1uhVqsLfP7MzEzh5eUlBg8erLd9+vTpws7OTty+fTvP/bRarcjMzBTr168X1tbW4u7du7rHgoODDZ5XrVaL4OBg3fqMGTOEQqEQsbGxeu06dOhg8G+TW85n4tq1awKA+OGHH3SPffLJJwKAiI+PN9gvKChIBAUF6dZXrlyZ57/3okWLBACxa9cu3TYAwsvLS6Slpem2JScnCysrK7Fw4cI868xPzr/3yJEjRcOGDXXbf/vtNwFAfPDBB/nu+88//whra2sxZMiQAp/j2fc6x7PvQc7PQevWrY2u+9nP6fr16wUAsXr16nz31Wq1olKlSqJnz55627t06SIqV66s97kly8ZhLHrplC1bFu3atTPY/s8//2Dw4MHw9vaGtbU1bG1tERQUBAC4cOFCocdt0KABKlasqFu3t7dHtWrVjPrNV6FQoEePHnrb6tWrp7fvwYMH4eLiYjA5etCgQYUeP8fKlSvRqFEj2Nvbw8bGBra2tti7d2+er69bt26wtrbWqweArqa4uDjcvHkTgwcP1hvWU6vVCAwMLLQWGxsbDB06FNHR0UhNTQUAaLVabNiwAT179oS7u7uu7Z9//onXX38d7u7uun+bYcOGQavV4u+//zb69QPA/v37Ubt2bdSvX19v++DBgw3apqSkYOzYsfDz89O9X2q1GoBxn4m87Nu3D05OTujXr5/e9pzhn7179+ptb9u2LVxcXHTrXl5e8PT0NOpz9d1336Fly5ZwdnbW1b9mzRq92nfs2AEAGD9+fL7H2b17N7RabYFtnkffvn3z3G7M53THjh2wt7fXDUPnxcrKChMmTMDPP/+MhIQEAFJv3a+//opx48Y991mVVPow7NBLJ2eYIbeMjAy0atUKf/zxB+bPn48DBw4gJiYG0dHRAIBHjx4VetzcX845lEqlUfs6OjrC3t7eYN/Hjx/r1u/cuQMvLy+DffPalpelS5fi7bffRrNmzbB161YcO3YMMTEx6Ny5c541Pvt6lEolgKfvxZ07dwAA3t7eBvvmtS0vI0aMwOPHjxEVFQUA2LlzJ5KSkjB8+HBdm4SEBLRq1Qo3btzAZ599hkOHDiEmJgZffvmlXj3GunPnjlE1Z2dno2PHjoiOjsb06dOxd+9eHD9+XDdvqajP++zzP/tF6+npCRsbG937muN5P1fR0dF44403UKFCBWzcuBFHjx5FTEyM7j3PcevWLVhbWxf4b5YztPS8E/Pzk9fPorGf01u3bsHHx8eo4VIHBwesXLkSgDQ86+DgUGBIIsvDOTv00snrt7l9+/bh5s2bOHDggK43B0Ce100xF3d3dxw/ftxge3JyslH7b9y4EW3atEFERITe9rwm5hpbT37Pb2xNtWrVwiuvvILIyEiMGTMGkZGR8PHxQceOHXVttm/fjgcPHiA6OlrXqwIAsbGxz123MTWfPXsWp0+fxtq1axEcHKzbfvny5ed63tzP/8cff0AIofdZTElJQVZWFsqXL/9Cx8+xceNGBAQEYMuWLXrP8+wkaA8PD2i1WiQnJ+cZPnLaANIEeT8/v3yf097e3uD4AHD79u08X1deP4vGfk49PDxw+PBhZGdnFxh43NzcEBwcjP/+97+YOnUqIiMjMXjwYINLBJBlY88OEZ7+p5vTe5HjP//5jznKyVNQUBDS09N1ww45cnpFCqNQKAxe319//WVwfSJjVa9eHSqVCps3b9Y72+XatWu6s2GMMXz4cPzxxx84fPgwfvrpJwQHB+sNn+X1byOEwOrVq5+r7rZt2+LcuXM4ffq03vZNmzbprRflM/Fsr1dB2rdvj4yMDGzfvl1ve85ZbO3bty/0GMZQKBSws7PTCxTJyckGZ2N16dIFAAzCRW4dO3aEtbV1gW0A6Wysv/76S2/b33//jbi4uCLVbczntEuXLnj8+HGhZyECwKRJk3D79m3069cP9+/fz3MyOlk29uwQAQgMDETZsmUxduxYzJkzB7a2tvjmm28MvhDNKTg4GMuWLcPQoUMxf/58VKlSBTt27MDOnTsBoNDu/O7du2PevHmYM2cOgoKCEBcXh48++ggBAQHIysoqcj1WVlaYN28eRo0ahd69e2P06NG4f/8+wsLCjB7GAqQ5R++++y4GDRoEjUZjcOpyhw4dYGdnh0GDBmH69Ol4/PgxIiIicO/evSLXDAChoaH4+uuv0a1bN8yfP193NtbFixf12tWoUQOVK1fGe++9ByEEypUrh59++gm7d+82OGbdunUBAJ999hmCg4Nha2uL6tWr6821yTFs2DB8+eWXCA4OxtWrV1G3bl0cPnwY4eHh6Nq1q95ZQy+ie/fuiI6Oxrhx49CvXz8kJiZi3rx5UKlUuHTpkq5dq1at8Oabb2L+/Pn4999/0b17dyiVSvz5559wdHTExIkT4e/vj/fffx/z5s3Do0ePMGjQILi5ueH8+fO4ffs25s6dCwB48803MXToUIwbNw59+/bFtWvXsHjxYl3PkLF1G/M5HTRoECIjIzF27FjExcWhbdu2yM7Oxh9//IGaNWti4MCBurbVqlVD586dsWPHDrz66qsG87XoJWDe+dFExSe/s7Fq166dZ/sjR46IFi1aCEdHR+Hh4SFGjRolTp06ZXCmU35nY3Xr1s3gmPmdhfLs2VjP1pnf8yQkJIg+ffoIZ2dn4eLiIvr27Sv+97//GZwdlBeNRiOmTp0qKlSoIOzt7UWjRo3E9u3bDc5gyjkb65NPPjE4BgAxZ84cvW3//e9/RdWqVYWdnZ2oVq2a+Prrr/M8K6oggwcPFgBEy5Yt83z8p59+EvXr1xf29vaiQoUKYtq0aWLHjh15vpeFnY0lhBDnz58XHTp0EPb29qJcuXJi5MiR4ocffjA4Xk47FxcXUbZsWdG/f3+RkJCQ5/swc+ZM4ePjI6ysrPSO8+xnQAgh7ty5I8aOHStUKpWwsbERarVazJw5Uzx+/FivHQAxfvx4g/cjv7OenvXxxx8Lf39/oVQqRc2aNcXq1avz/FxptVqxbNkyUadOHWFnZyfc3NxEixYtxE8//aTXbv369aJp06bC3t5eODs7i4YNG+r9bGRnZ4vFixeLSpUqCXt7e9GkSROxb9++fH8OvvvuO4Oajf2cCiGd8Th79mzd58/d3V20a9dOHDlyxOC4a9euFQBEVFRUoe8bWR6FEM9cbYmISpXw8HB8+OGHSEhIMPkEUiJL0bdvXxw7dgxXr16Fra2tucuhEsZhLKJSZMWKFQCkIZbMzEzs27cPn3/+OYYOHcqgQ/QMjUaDU6dO4fjx49i2bRuWLl3KoPOSYtghKkUcHR2xbNkyXL16FRqNBhUrVsSMGTPw4Ycfmrs0ItlJSkpCYGAgXF1dMWbMGEycONHcJZGZcBiLiIiILBpPPSciIiKLxrBDREREFo1hh4iIiCwaJyhDugfOzZs34eLiwhvDERERlRJCCKSnpxd6nzSGHQA3b94s8H4vREREJF+JiYkFXn6DYQfQXdI9MTERrq6uZq6GiIiIjJGWlgY/P788b82SG8MOnt7wz9XVlWGHiIiolClsCgonKBMREZFFY9ghIiIii8awQ0RERBbNrHN2fvvtN3zyySc4efIkkpKSsG3bNvTq1Uv3uBACc+fOxapVq3Dv3j00a9YMX375JWrXrq1ro9FoMHXqVGzevBmPHj1C+/bt8dVXXxXLTRG1Wi0yMzNNflyyfLa2trC2tjZ3GURELyWzhp0HDx6gfv36GD58OPr27Wvw+OLFi7F06VKsXbsW1apVw/z589GhQwfExcXpZl6Hhobip59+QlRUFNzd3TFlyhR0794dJ0+eNNmXixACycnJuH//vkmORy+nMmXKwNvbm9dyIiIqYbK5EahCodDr2RFCwMfHB6GhoZgxYwYAqRfHy8sLixYtwpgxY5CamgoPDw9s2LABAwYMAPD0mjn/+9//0KlTJ6OeOy0tDW5ubkhNTc3zbKykpCTcv38fnp6ecHR05JcVFYkQAg8fPkRKSgrKlCkDlUpl7pKIiCxCYd/fOWR76nl8fDySk5PRsWNH3TalUomgoCAcOXIEY8aMwcmTJ5GZmanXxsfHB3Xq1MGRI0eMDjsF0Wq1uqDj7u7+wsejl5ODgwMAICUlBZ6enhzSIiIqQbINO8nJyQAALy8vve1eXl64du2aro2dnR3Kli1r0CZn/7xoNBpoNBrdelpaWr5tc+boODo6Fu0FED0j5zOUmZnJsENEVIJkfzbWs0NGQohCh5EKa7Nw4UK4ubnpFmNuFcGhK3pR/AwREZmHbMOOt7c3ABj00KSkpOh6e7y9vfHkyRPcu3cv3zZ5mTlzJlJTU3VLYmKiiasnIiJLpdUCBw4AmzdLf2q15q6ICiPbsBMQEABvb2/s3r1bt+3Jkyc4ePAgAgMDAQCNGzeGra2tXpukpCScPXtW1yYvSqVSd2sI3iLCeG3atEFoaKjR7a9evQqFQoHY2Nhiq4mIqCRFRwP+/kDbtsDgwdKf/v7SdpIvs87ZycjIwOXLl3Xr8fHxiI2NRbly5VCxYkWEhoYiPDwcVatWRdWqVREeHg5HR0cMHjwYAODm5oaRI0diypQpcHd3R7ly5TB16lTUrVsXr732mrleVp60WuDQISApCVCpgFatgOKatlHYcElwcDDWrl1b5ONGR0fD1tbW6PZ+fn5ISkpC+fLli/xcRERyEx0N9OsHPHsO840b0vbvvwf69DFPbVQws4adEydOoG3btrr1d999F8DTL+Pp06fj0aNHGDdunO6igrt27dK7u+myZctgY2ODN954Q3dRwbVr18pqAmh0NDB5MnD9+tNtvr7AZ58Vzw9GUlKS7u9btmzB7NmzERcXp9uWc2ZQjszMTKNCTLly5YpUh7W1tW44koioNNNqpf/H87pYixCAQgGEhgI9exbfL7L0AgSJ1NRUAUCkpqYaPPbo0SNx/vx58ejRo+c69tatQigUQkg/Dk8XhUJatm590eoLFhkZKdzc3HTr8fHxAoDYsmWLCAoKEkqlUnz99dfi9u3bYuDAgaJChQrCwcFB1KlTR2zatEnvWEFBQWLy5Mm6dbVaLRYsWCCGDx8unJ2dhZ+fn/jPf/5j8Fx//vmnEEKI/fv3CwBiz549onHjxsLBwUG0aNFCXLx4Ue955s2bJzw8PISzs7MYOXKkmDFjhqhfv36+rzErK0uMGDFC+Pv7C3t7e1GtWjWxfPlyg3Zr1qwRtWrVEnZ2dsLb21uMHz9e99i9e/fE6NGjhaenp1AqlaJ27drip59+MuIdNt6LfpaIyHz27zf8fzyvZf9+c1f6cino+zs32c7ZsQSF/SYASL8JmGNy24wZMzBp0iRcuHABnTp1wuPHj9G4cWP8/PPPOHv2LN566y28+eab+OOPPwo8zpIlS9CkSRP8+eefGDduHN5++21cvHixwH0++OADLFmyBCdOnICNjQ1GjBihe+ybb77BggULsGjRIpw8eRIVK1ZEREREgcfLzs6Gr68vvv32W5w/fx6zZ8/G+++/j2+//VbXJiIiAuPHj8dbb72FM2fO4Mcff0SVKlV0+3fp0gVHjhzBxo0bcf78eXz88cey6h0kKm6cdFuwXB3mJmn3spDN56qEwpesFVfPjhx+E8ivZyevno9nde3aVUyZMkW3nlfPztChQ3Xr2dnZwtPTU0REROg9V149Ozl++eUXAUD3/jZr1kyvx0UIIVq2bFlgz05exo0bJ/r27atb9/HxER988EGebXfu3CmsrKxEXFxckZ6jqNizQ3K1dasQvr76/y/5+hZ/z3NpIof/z0ubkvhcsWdHBuT8m0CTJk301rVaLRYsWIB69erB3d0dzs7O2LVrFxISEgo8Tr169XR/VygU8Pb2RkpKitH75Nw6IWefuLg4vPLKK3rtn13Py8qVK9GkSRN4eHjA2dkZq1ev1tWekpKCmzdvon379nnuGxsbC19fX1SrVq3Q5yGyNDmTbnPPKQSeTrrlWUaSVq2kuZb5nf+hUAB+flI7kt/nimGnGBl7CyRz3CrJyclJb33JkiVYtmwZpk+fjn379iE2NhadOnXCkydPCjzOsxObFQoFsrOzjd4n58yx3PvkdSHJgnz77bd45513MGLECOzatQuxsbEYPny4rvZnJ2Q/q7DHiSyVnIfa5cbaWjqpBDAMPDnry5dzcjIgz88Vw04xKk2/CRw6dAg9e/bE0KFDUb9+fVSqVAmXLl0q8TqqV6+O48eP6207ceJEgfscOnQIgYGBGDduHBo2bIgqVargypUrusddXFzg7++PvXv35rl/vXr1cP36dfz9998v/gKISpFDhwx/885NCCAxUWpH0tmz338PVKigv93Xl6ed5ybHzxXDTjEqTb8JVKlSBbt378aRI0dw4cIFjBkzpsD7ixWXiRMnYs2aNVi3bh0uXbqE+fPn46+//irw2kFVqlTBiRMnsHPnTvz999+YNWsWYmJi9NqEhYVhyZIl+Pzzz3Hp0iWcOnUKX3zxBQAgKCgIrVu3Rt++fbF7927Ex8djx44d+PXXX4v1tRKZm5yH2uWqTx/g6lVg/35g0ybpz/h4Bp3c5Pi5YtgpZqXlN4FZs2ahUaNG6NSpE9q0aQNvb2/06tWrxOsYMmQIZs6cialTp6JRo0aIj49HSEgI7O3t891n7Nix6NOnDwYMGIBmzZrhzp07GDdunF6b4OBgLF++HF999RVq166N7t276/Vcbd26FU2bNsWgQYNQq1YtTJ8+HVr23ZOFk/NQu5xZWwNt2gCDBkl/yuEXVjmR4+dKIQqbEPESSEtLg5ubG1JTUw1uHfH48WPEx8cjICCgwC/cwpTkFZQtTYcOHeDt7Y0NGzaYu5QXYqrPEpGpaLXSrQ5u3Mh7foVCIf1iFh/P/6/IeCX5uSro+zs3s15B+WWS85sAFezhw4dYuXIlOnXqBGtra2zevBl79uzRu/8ZEZlGzlB7v37SF1DuLya5DbVT6SHHzxWHsUhWFAoF/ve//6FVq1Zo3LgxfvrpJ2zdulV29zojshSlZaidShe5fa44jIWSGcYi4meJ5IxD7VQcivtzxWEsIiIyGofaqTjI5XPFYSwiIiKyaAw7REREZNEYdoiIiMiiMewQERGRRWPYISIiIovGsEP5atOmDUJDQ3Xr/v7+WL58eYH7KBQKbN++/YWf21THISIiYtixQD169Mj3InxHjx6FQqHAqVOninzcmJgYvPXWWy9anp6wsDA0aNDAYHtSUhK6dOli0uciIqKXE8OOBRo5ciT27duHa9euGTz29ddfo0GDBmjUqFGRj+vh4QFHR0dTlFgob29vKJXKEnkuIiKybAw7Fqh79+7w9PTE2rVr9bY/fPgQW7ZswciRI3Hnzh0MGjQIvr6+cHR0RN26dbF58+YCj/vsMNalS5fQunVr2Nvbo1atWnnev2rGjBmoVq0aHB0dUalSJcyaNQuZmZkAgLVr12Lu3Lk4ffo0FAoFFAqFruZnh7HOnDmDdu3awcHBAe7u7njrrbeQkZGhezwkJAS9evXCp59+CpVKBXd3d4wfP173XHm5cuUKevbsCS8vLzg7O6Np06bYs2ePXhuNRoPp06fDz88PSqUSVatWxZo1a3SPnzt3Dt26dYOrqytcXFzQqlUrXLlypcD3kUqOVgscOABs3iz9yRvZE72ceAXlIhICePjQPM/t6Pj0JmoFsbGxwbBhw7B27VrMnj0biv/f6bvvvsOTJ08wZMgQPHz4EI0bN8aMGTPg6uqKX375BW+++SYqVaqEZs2aFfoc2dnZ6NOnD8qXL49jx44hLS1Nb35PDhcXF6xduxY+Pj44c+YMRo8eDRcXF0yfPh0DBgzA2bNn8euvv+pChpubm8ExHj58iM6dO6N58+aIiYlBSkoKRo0ahQkTJugFuv3790OlUmH//v24fPkyBgwYgAYNGmD06NF5voaMjAx07doV8+fPh729PdatW4cePXogLi4OFStWBAAMGzYMR48exeeff4769esjPj4et2/fBgDcuHEDrVu3Rps2bbBv3z64urri999/R1ZWVqHvHxW/6Ghg8mTg+vWn23x9pRsU8n5PRC8ZQSI1NVUAEKmpqQaPPXr0SJw/f148evRICCFERoYQUuQp+SUjw/jXdOHCBQFA7Nu3T7etdevWYtCgQfnu07VrVzFlyhTdelBQkJg8ebJuXa1Wi2XLlgkhhNi5c6ewtrYWiYmJusd37NghAIht27bl+xyLFy8WjRs31q3PmTNH1K9f36Bd7uOsWrVKlC1bVmTkegN++eUXYWVlJZKTk4UQQgQHBwu1Wi2ysrJ0bfr37y8GDBiQby15qVWrlvjiiy+EEELExcUJAGL37t15tp05c6YICAgQT548MerYz36WqPhs3SqEQmH4M6RQSMvWreaukIhMoaDv79w4jGWhatSogcDAQHz99dcApCGbQ4cOYcSIEQAArVaLBQsWoF69enB3d4ezszN27dqFhIQEo45/4cIFVKxYEb6+vrptLVq0MGj3/fff49VXX4W3tzecnZ0xa9Yso58j93PVr18fTk5Oum0tW7ZEdnY24uLidNtq164N61x3mFOpVEhJScn3uA8ePMD06dNRq1YtlClTBs7Ozrh48aKuvtjYWFhbWyMoKCjP/WNjY9GqVSvY2toW6fVQ8dJqpR6dvG5xnLMtNJRDWkQvEw5jFZGjI5BrqkiJP3dRjBw5EhMmTMCXX36JyMhIqNVqtG/fHgCwZMkSLFu2DMuXL0fdunXh5OSE0NBQPHnyxKhjizy+SRTPjLEdO3YMAwcOxNy5c9GpUye4ubkhKioKS5YsKdLrEEIYHDuv53w2dCgUCmRnZ+d73GnTpmHnzp349NNPUaVKFTg4OKBfv36698DBwaHAugp7nMzj0CH9oatnCQEkJkrt5HCDQiIqfgw7RaRQALk6GGTtjTfewOTJk7Fp0yasW7cOo0eP1oWDQ4cOoWfPnhg6dCgAaQ7OpUuXULNmTaOOXatWLSQkJODmzZvw8fEBIJ3Wntvvv/8OtVqNDz74QLft2TPE7OzsoC3kV+xatWph3bp1ePDgga535/fff4eVlRWqVatmVL15OXToEEJCQtC7d28A0hyeq1ev6h6vW7cusrOzcfDgwTxP5a9Xrx7WrVuHzMxM9u7ISFKSadsRUenHYSwL5uzsjAEDBuD999/HzZs3ERISonusSpUq2L17N44cOYILFy5gzJgxSE5ONvrYr732GqpXr45hw4bh9OnTOHTokF6oyXmOhIQEREVF4cqVK/j888+xbds2vTb+/v6Ij49HbGwsbt++DY1GY/BcQ4YMgb29PYKDg3H27Fns378fEydOxJtvvgkvL6+ivSnP1BcdHY3Y2FicPn0agwcP1usJ8vf3R3BwMEaMGIHt27cjPj4eBw4cwLfffgsAmDBhAtLS0jBw4ECcOHECly5dwoYNG/SG1qjkqVSmbUdEpR/DjoUbOXIk7t27h9dee013hhEAzJo1C40aNUKnTp3Qpk0beHt7o1evXkYf18rKCtu2bYNGo8Err7yCUaNGYcGCBXptevbsiXfeeQcTJkxAgwYNcOTIEcyaNUuvTd++fdG5c2e0bdsWHh4eeZ7+7ujoiJ07d+Lu3bto2rQp+vXrh/bt22PFihVFezOesWzZMpQtWxaBgYHo0aMHOnXqZHD9oYiICPTr1w/jxo1DjRo1MHr0aDx48AAA4O7ujn379iEjIwNBQUFo3LgxVq9ezV4eM2vVSjrrKr8zFxUKwM9PakdELweFyGvyxUsmLS0Nbm5uSE1Nhaurq95jjx8/Rnx8PAICAmBvb2+mCskS8LNUcqKjgX79pL/n/h8uJwB9/z1PPyeyBAV9f+fGnh0isjh9+kiBpkIF/e2+vgw6RC8jTlAmIovUpw/Qs6d01lVSkjRHp1UrINfVCYjoJcGwQ0QWy9qap5cTEYexiIiIyMLJPuykp6cjNDQUarUaDg4OCAwMRExMjO7xkJAQ3U0kc5bmzZubvA7O46YXxc8QEZF5yH4Ya9SoUTh79iw2bNgAHx8fbNy4Ea+99hrOnz+PCv8/+7Bz586IjIzU7WNnZ2ey5885jfjhw4e8Yi69kIf/fwdZnppORFSyZB12Hj16hK1bt+KHH35A69atAQBhYWHYvn07IiIiMH/+fACAUqmEt7d3sdRgbW2NMmXK6O6x5OjomO+tC4jyIoTAw4cPkZKSgjJlyujdv4uIiIqfrMNOVlYWtFqtwTVJHBwccPjwYd36gQMH4OnpiTJlyiAoKAgLFiyAp6dnvsfVaDR6V+pNS0srsI6cIFXQTSWJClOmTJliC+VERJQ/2V9UMDAwEHZ2dti0aRO8vLywefNmDBs2DFWrVkVcXBy2bNkCZ2dnqNVqxMfHY9asWcjKysLJkyehVCrzPGZYWBjmzp1rsL2wixJptVpkZmaa7LXRy8PW1pY9OkREJmbsRQVlH3auXLmCESNG4LfffoO1tTUaNWqEatWq4dSpUzh//rxB+6SkJKjVakRFRaFPPlcOy6tnx8/Pr9A3i4iIiOTD2LAj62EsAKhcuTIOHjyIBw8eIC0tDSqVCgMGDEBAQECe7VUqFdRqNS5dupTvMZVKZb69PkRERGRZZH/qeQ4nJyeoVCrcu3cPO3fuRM+ePfNsd+fOHSQmJkLFWxoTERERSkHPzs6dOyGEQPXq1XH58mVMmzYN1atXx/Dhw5GRkYGwsDD07dsXKpUKV69exfvvv4/y5cujd+/e5i6diIiIZED2YSc1NRUzZ87E9evXUa5cOfTt2xcLFiyAra0tsrKycObMGaxfvx7379+HSqVC27ZtsWXLFri4uJi7dCIiIpIB2U9QLgnGTnAiIiIi+TD2+7vUzNkhIiIieh4MO0RERGTRGHaIiIjIojHsEBERkUVj2CEiIiKLxrBDREREFo1hh4iIiCwaww4RERFZNIYdIiIismgMO0RERGTRGHaIiIjIojHsEBERkUVj2CEiIiKLxrBDREREFo1hh4iIiCwaww4RERFZNIYdIiIismgMO0RERGTRGHaIiIjIojHsEBERkUVj2CEiIiKLxrBDREREFo1hh4iIiCwaww4RERFZNIYdIiIismgMO0RERGTRGHaIiIjIojHsEBERkUVj2CEiIiKLxrBDREREFo1hh4iIiCwaww4RERFZNIYdIiIismiyDzvp6ekIDQ2FWq2Gg4MDAgMDERMTo3tcCIGwsDD4+PjAwcEBbdq0wblz58xYMREREcmJ7MPOqFGjsHv3bmzYsAFnzpxBx44d8dprr+HGjRsAgMWLF2Pp0qVYsWIFYmJi4O3tjQ4dOiA9Pd3MlRMREZEcKIQQwtxF5OfRo0dwcXHBDz/8gG7duum2N2jQAN27d8e8efPg4+OD0NBQzJgxAwCg0Wjg5eWFRYsWYcyYMUY9T1paGtzc3JCamgpXV9dieS1ERERkWsZ+f8u6ZycrKwtarRb29vZ62x0cHHD48GHEx8cjOTkZHTt21D2mVCoRFBSEI0eO5HtcjUaDtLQ0vYWIiIgsk6zDjouLC1q0aIF58+bh5s2b0Gq12LhxI/744w8kJSUhOTkZAODl5aW3n5eXl+6xvCxcuBBubm66xc/Pr1hfBxEREZmPrMMOAGzYsAFCCFSoUAFKpRKff/45Bg8eDGtra10bhUKht48QwmBbbjNnzkRqaqpuSUxMLLb6iYiIyLxkH3YqV66MgwcPIiMjA4mJiTh+/DgyMzMREBAAb29vADDoxUlJSTHo7clNqVTC1dVVbyEiIiLLJPuwk8PJyQkqlQr37t3Dzp070bNnT13g2b17t67dkydPcPDgQQQGBpqxWiIiIpILG3MXUJidO3dCCIHq1avj8uXLmDZtGqpXr47hw4dDoVAgNDQU4eHhqFq1KqpWrYrw8HA4Ojpi8ODB5i6diIiIZED2YSc1NRUzZ87E9evXUa5cOfTt2xcLFiyAra0tAGD69Ol49OgRxo0bh3v37qFZs2bYtWsXXFxczFw5ERERyYGsr7NTUnidHSIiotLHIq6zQ0RERPSiGHaIiIjIojHsEBERkUVj2CEiIiKLxrBDREREFk32p54TkT6tFjh0CEhKAlQqoFUrINfdU4iI6BkMO0SlSHQ0MHkycP36022+vsBnnwF9+pivLiIiOeMwFlEpER0N9OunH3QA4MYNaXt0tHnqIiKSO4YdolJAq5V6dPK6BGjOttBQqR0REelj2CEqBQ4dMuzRyU0IIDFRakdERPoYdohKgaQk07YjInqZMOwQlQIqlWnbERG9TBh2iEqBVq2ks64UirwfVygAPz+pHRER6WPYISoFrK2l08sBw8CTs758Oa+3Q0SUF4YdolKiTx/g+++BChX0t/v6Stt5nR0iorzxooJEpUifPkDPnryCMhFRUTDsEJUy1tZAmzbmroKIqPTgMBYRERFZNIYdIiIismgMO0RERGTRGHaIiIjIojHsEBERkUVj2CEiIiKLxrBDREREFo1hh4iIiCwaww4RERFZNIYdIiIismgMO0RERGTRGHaIiIjIojHsEBERkUVj2CEiIiKLxrBDREREFk3WYScrKwsffvghAgIC4ODggEqVKuGjjz5Cdna2rk1ISAgUCoXe0rx5czNWTURERHJiY+4CCrJo0SKsXLkS69atQ+3atXHixAkMHz4cbm5umDx5sq5d586dERkZqVu3s7MzR7lEREQkQ7IOO0ePHkXPnj3RrVs3AIC/vz82b96MEydO6LVTKpXw9vY2R4lEREQkc7Iexnr11Vexd+9e/P333wCA06dP4/Dhw+jatateuwMHDsDT0xPVqlXD6NGjkZKSUuBxNRoN0tLS9BYiIiKyTLLu2ZkxYwZSU1NRo0YNWFtbQ6vVYsGCBRg0aJCuTZcuXdC/f3+o1WrEx8dj1qxZaNeuHU6ePAmlUpnncRcuXIi5c+eW1MsgIiIiM1IIIYS5i8hPVFQUpk2bhk8++QS1a9dGbGwsQkNDsXTpUgQHB+e5T1JSEtRqNaKiotCnT58822g0Gmg0Gt16Wloa/Pz8kJqaCldX12J5LURERGRaaWlpcHNzK/T7W9Y9O9OmTcN7772HgQMHAgDq1q2La9euYeHChfmGHZVKBbVajUuXLuV7XKVSmW+vDxEREVkWWc/ZefjwIays9Eu0trbWO/X8WXfu3EFiYiJUKlVxl0dERESlgKx7dnr06IEFCxagYsWKqF27Nv78808sXboUI0aMAABkZGQgLCwMffv2hUqlwtWrV/H++++jfPny6N27t5mrJyIiIjmQddj54osvMGvWLIwbNw4pKSnw8fHBmDFjMHv2bABSL8+ZM2ewfv163L9/HyqVCm3btsWWLVvg4uJi5uqJiIhIDmQ9QbmkGDvBiYiIiOTD2O9vWc/ZISIiInpRDDtERERk0Rh2iIiIyKIx7BAREZFFY9ghIiIii8awQ0RERBaNYYeIiIgsGsMOERERWbQihx1/f3989NFHSEhIKI56iIiIiEyqyGFnypQp+OGHH1CpUiV06NABUVFR0Gg0xVEbERER0QsrctiZOHEiTp48iZMnT6JWrVqYNGkSVCoVJkyYgFOnThVHjURERETP7YXvjZWZmYmvvvoKM2bMQGZmJurUqYPJkydj+PDhUCgUpqqzWPHeWERERKWPsd/fz33X88zMTGzbtg2RkZHYvXs3mjdvjpEjR+LmzZv44IMPsGfPHmzatOl5D09ERERkEkUOO6dOnUJkZCQ2b94Ma2trvPnmm1i2bBlq1Kiha9OxY0e0bt3apIUSERERPY8ih52mTZuiQ4cOiIiIQK9evWBra2vQplatWhg4cKBJCiQiIiJ6EUUOO//88w/UanWBbZycnBAZGfncRRERERGZSpHPxkpJScEff/xhsP2PP/7AiRMnTFIUERERkakUOeyMHz8eiYmJBttv3LiB8ePHm6QoIiIiIlMpctg5f/48GjVqZLC9YcOGOH/+vEmKIiIiIjKVIocdpVKJf//912B7UlISbGye+0x2IiIiomJR5LDToUMHzJw5E6mpqbpt9+/fx/vvv48OHTqYtDgiIiKiF1XkrpglS5agdevWUKvVaNiwIQAgNjYWXl5e2LBhg8kLJCIiInoRRQ47FSpUwF9//YVvvvkGp0+fhoODA4YPH45Bgwblec0dIiIiInN6rkk2Tk5OeOutt0xdCxEREZHJPfeM4vPnzyMhIQFPnjzR2/7666+/cFFEREREpvJcV1Du3bs3zpw5A4VCgZybpufc4Vyr1Zq2QiIiIqIXUOSzsSZPnoyAgAD8+++/cHR0xLlz5/Dbb7+hSZMmOHDgQDGUSERERPT8ityzc/ToUezbtw8eHh6wsrKClZUVXn31VSxcuBCTJk3Cn3/+WRx1EhERET2XIvfsaLVaODs7AwDKly+PmzdvAgDUajXi4uJMWx0RERHRCypyz06dOnXw119/oVKlSmjWrBkWL14MOzs7rFq1CpUqVSqOGomIiIieW5HDzocffogHDx4AAObPn4/u3bujVatWcHd3x5YtW0xeIBEREdGLUIic06lewN27d1G2bFndGVmlTVpaGtzc3JCamgpXV1dzl0NERERGMPb7u0hzdrKysmBjY4OzZ8/qbS9XrlypDTpERERk2YoUdmxsbKBWq0vsWjpZWVn48MMPERAQAAcHB1SqVAkfffQRsrOzdW2EEAgLC4OPjw8cHBzQpk0bnDt3rkTqIyIiIvkr8tlYH374IWbOnIm7d+8WRz16Fi1ahJUrV2LFihW4cOECFi9ejE8++QRffPGFrs3ixYuxdOlSrFixAjExMfD29kaHDh2Qnp5e7PURERGR/BV5zk7Dhg1x+fJlZGZmQq1Ww8nJSe/xU6dOmay47t27w8vLC2vWrNFt69u3LxwdHbFhwwYIIeDj44PQ0FDMmDEDAKDRaODl5YVFixZhzJgxRj0P5+wQERGVPsZ+fxf5bKxevXq9SF1F8uqrr2LlypX4+++/Ua1aNZw+fRqHDx/G8uXLAQDx8fFITk5Gx44ddfsolUoEBQXhyJEj+YYdjUYDjUajW09LSyvW10FERETmU+SwM2fOnOKoI08zZsxAamoqatSoAWtra2i1WixYsACDBg0CACQnJwMAvLy89Pbz8vLCtWvX8j3uwoULMXfu3OIrnIiIiGSjyHN2StKWLVuwceNGbNq0CadOncK6devw6aefYt26dXrtnj0TTAhR4NlhM2fORGpqqm5JTEwslvqJiIjI/Ircs2NlZVVgkDDlmVrTpk3De++9h4EDBwIA6tati2vXrmHhwoUIDg6Gt7c3AKmHR6VS6fZLSUkx6O3JTalUQqlUmqxOIiIikq8ih51t27bprWdmZuLPP//EunXrTD409PDhQ1hZ6Xc+WVtb6049DwgIgLe3N3bv3o2GDRsCAJ48eYKDBw9i0aJFJq2FiIiISqcih52ePXsabOvXrx9q166NLVu2YOTIkSYpDAB69OiBBQsWoGLFiqhduzb+/PNPLF26FCNGjAAgDV+FhoYiPDwcVatWRdWqVREeHg5HR0cMHjzYZHUQERFR6WWS20UAwJUrV1CvXj3dfbNMIT09HbNmzcK2bduQkpICHx8fDBo0CLNnz4adnR0AaX7O3Llz8Z///Af37t1Ds2bN8OWXX6JOnTpGPw9PPSciIip9jP3+NknYefToEWbOnIkdO3YgLi7uRQ9X4hh2iIiISp9iu87Oszf8FEIgPT0djo6O2Lhx4/NVS0RERFRMihx2li1bphd2rKys4OHhgWbNmqFs2bImLY6IiIjoRRU57ISEhBRDGURERETFo8gXFYyMjMR3331nsP27774zuNgfERERkbkVOex8/PHHKF++vMF2T09PhIeHm6QoIiIiIlMpcti5du0aAgICDLar1WokJCSYpCgiIiIiUyly2PH09MRff/1lsP306dNwd3c3SVFEREREplLksDNw4EBMmjQJ+/fvh1arhVarxb59+zB58mTdPayIiIiI5KLIZ2PNnz8f165dQ/v27WFjI+2enZ2NYcOGcc4OERERyc5zX0H50qVLiI2NhYODA+rWrQu1Wm3q2koMr6BMRERU+hTbFZRz5Nx4k4iIiEjOijxnp1+/fvj4448Ntn/yySfo37+/SYoiIiIiMpUih52DBw+iW7duBts7d+6M3377zSRFEREREZlKkcNORkYG7OzsDLbb2toiLS3NJEURERERmUqRw06dOnWwZcsWg+1RUVGoVauWSYoiIiIiMpUiT1CeNWsW+vbtiytXrqBdu3YAgL1792LTpk34/vvvTV4gERER0Ysocth5/fXXsX37doSHh+P777+Hg4MD6tevj3379vG0bSIiIpKd577OTo779+/jm2++wZo1a3D69GlotVpT1VZieJ0dIiKi0sfY7+8iz9nJsW/fPgwdOhQ+Pj5YsWIFunbtihMnTjzv4eglp9UCBw4AmzdLf5bCzExERDJVpGGs69evY+3atfj666/x4MEDvPHGG8jMzMTWrVs5OZmeW3Q0MHkycP36022+vsBnnwF9+pivLiIisgxG9+x07doVtWrVwvnz5/HFF1/g5s2b+OKLL4qzNnoJREcD/frpBx0AuHFD2h4dbZ66iIjIchgddnbt2oVRo0Zh7ty56NatG6ytrYuzLnoJaLVSj05es8ZytoWGckiLiIhejNFh59ChQ0hPT0eTJk3QrFkzrFixArdu3SrO2sjCHTpk2KOTmxBAYqLUjoiI6HkZHXZatGiB1atXIykpCWPGjEFUVBQqVKiA7Oxs7N69G+np6cVZJ1mgpCTTtiMiIspLkc/GcnR0xIgRI3D48GGcOXMGU6ZMwccffwxPT0+8/vrrxVEjWSiVyrTtiIiI8vLcp54DQPXq1bF48WJcv34dmzdvNlVN9JJo1Uo660qhyPtxhQLw85PaERERPa8XCjs5rK2t0atXL/z444+mOBy9JKytpdPLAcPAk7O+fLnUjoiI6HmZJOwQPa8+fYDvvwcqVNDf7usrbed1doiI6EUV+d5YRKbWpw/Qs6d01lVSkjRHp1Ur9ugQEZFpMOyQLFhbA23amLsKIiKyRBzGIiIiIovGsENEREQWTfZhx9/fHwqFwmAZP348ACAkJMTgsebNm5u5aiIiIpIL2c/ZiYmJgTbXzZHOnj2LDh06oH///rptnTt3RmRkpG7dzs6uRGskIiIi+ZJ92PHw8NBb//jjj1G5cmUEBQXptimVSnh7e5d0aURERFQKyH4YK7cnT55g48aNGDFiBBS5rkJ34MABeHp6olq1ahg9ejRSUlLMWCURERHJiex7dnLbvn077t+/j5CQEN22Ll26oH///lCr1YiPj8esWbPQrl07nDx5EkqlMs/jaDQaaDQa3XpaWlpxl05ERERmohBCCHMXYaxOnTrBzs4OP/30U75tkpKSoFarERUVhT75XH43LCwMc+fONdiempoKV1dXk9VLRERExSctLQ1ubm6Ffn+XmmGsa9euYc+ePRg1alSB7VQqFdRqNS5dupRvm5kzZyI1NVW3JCYmmrpcIiIikolSM4wVGRkJT09PdOvWrcB2d+7cQWJiIlQqVb5tlEplvkNcREREZFlKRc9OdnY2IiMjERwcDBubp/ksIyMDU6dOxdGjR3H16lUcOHAAPXr0QPny5dG7d28zVkxERERyUSp6dvbs2YOEhASMGDFCb7u1tTXOnDmD9evX4/79+1CpVGjbti22bNkCFxcXM1VLREREclKqJigXF2MnOBEREZF8WNwEZSIiIqLnwbBDREREFo1hh4iIiCwaww4RERFZNIYdIiIismgMO0RERGTRGHaIiIjIojHsEBERkUVj2CEiIiKLxrBDRBbt/n3gzh2A14onenmVintjEREZ48kTIDYW+OOPp8vly9JjdnaAj4/+olIZbnNzAxQKs74MKgWePAEyMwEnJ3NXQsZg2CGiUkkIID5eP9icOiV9CeXlyRPg6lVpKYiDQ8FhKGfhvYYtixBAaiqQkiIt//779O/Prv/7r9RjCAAeHkBAAFCpkv4SEAD4+gI2/JaVBf4zEFGpcP8+cPz402Bz/Dhw65ZhO3d34JVXgObNgWbNgKZNAWdnIDkZuHmz4OXePeDRI+DKFWkpiLNzwWEoJyzxN3/zyczUDyx5hZbcj+UXlAty65a0HD9u+JiNDaBW6weg3IGobNkXf41kHN71HLzrOZHcZGYCZ84Ax449DTdxcYbtbG2Bhg2lUJOzVK78/MNQjx4VHoqSkqQeAGO5uhYchnL+dHB4vppfJkIAaWmFh5ac9Xv3iv4cLi6Alxfg6fl0yb2e++/W1lJP4T//PF3i46U/r14tPDyVKZN/r5BaLQ29UsGM/f5m2AHDDpE5CQEkJOgPR508CTx+bNi2cmX9YNOgAaBUlnjJePBACj35haGbN4EbN6R2xipbtvA5Rd7e5nm9xSkzU+oZKaznJefvRe19sbaWhpqMCTAeHqYLnVqt9DnIHYByL//+W/D+VlbSMFh+vUIeHpxbBjDsFAnDDlHJSUsDYmL0w01e//GXKSMNR+UEm1dekf6DL03S0wsORDmhKK9gl5/y5QufU+TlJfV6mYMQ0us2puclJQW4e7foz+HiUnivS8562bJScJCbBw/y7xX65x+pl7EgTk759woFBLw8PYUMO0XAsENUPLKygHPn9IPN+fOGp4Hb2AD16+v32lStKs8vKVPLmRibXxjKvRjbq6FQSF/2hc0pyhmKKUxmJnD7tnE9LykpgEZTtPcgp/eloNCSe93Sv8iFkN7P/HqFbtwo/FIKKlX+vUIqleX8bDHsFAHDDpFpXL+uH2xOnAAePjRsp1brB5tGjSz/C+xFCSH1ghQWiJKSpJBpDCsrKUjkDkC2toYh5nl6X5ydC+91yfl7uXKW8+VbEjQa4Nq1vHuE/vlH6j0tiFIJ+Pvn3StUqVLpOtOQYacIGHbMb+tWYMEC6Te8ChXyX/jPIx8ZGdLcmpxgc+yY9GX7LBcX/eGoZs2kLzoqHtnZUi9MfmEoZ0lOltoay8qqaHNfHB2L7zVS/nJCcV49QvHxUkjSags+Rvny+fcKye10eoadImDYMZ9Hj4B33wVWrjSuvbNzwWGoQgVpEqcxXfNkPK0WuHBBv9fm7FnDL0srK6BuXf1gU6MG/z3kSKuVJgbnDkA3bkjb8wo07H2xDFlZQGJi/r1Cd+4UvL+NDVCxYv69QmXLluzEaYadImDYMY/z54EBA6QvTYUCmDYNaNlS+g83r8XY032trKTAU1goKk1dtSUtOflpb03OcFR6umE7X1/9YNO4Ma8rQ1SapaXl3ysUH1/4vDE3t7x7hCpVKp7T6Rl2ioBhp2QJAaxZA0yaJPXseHkBGzYAHToUvF9GxtPfPvNbkpIK76LN4eJSeCDy8rL8XomHD6UrD+futUlIMGzn5AQ0aSKFmpwL9vn4lHy9RGQe2dlPT6fPq1coObng/Rcvln6pNSVjv79lNPJGL4PUVGDMGGDLFmm9Y0dg/Xrj5nA4OwPVqklLfrRaaVJlQYHoxg3pt5f0dODiRWnJj7V1/r1EPj6lr5coO1u6OF/uYPPXX4YBUaEAatfW77WpXdvygx8R5S/n2j++vkDr1oaPP3xoeDp97lBUqVKJl6zDnh2wZ6ekHD8ODBwoffBtbKQJyVOnmmceQEZG4YEoObn09xLduqUfbI4fz3s40Nv7aahp3lzqwSktAY6I5E8I6ZctU/8fyGGsImDYKV7Z2cCSJcD770uT4/z9gago6YtVzrRa6bTbwkJRXnNZ8lJQL1Huxdn5+ep9/Bj480/9cBMfb9jOwUGaW5O718bPj1djJaLSh8NYJAspKcCwYcDOndJ6//7AqlXS1XHlztr66bVHmjbNv116uvG9RDnrBXF1LTwQeXpK3cK5JxGfPi1d/O1ZNWo8nWPTrBlQp475rq5LRGQODDtUbPbsAd58U/qit7cHPv8cGDXK8noQXFykQFGjRv5tsrL0e4nym2idni7NJ0pLk071LioPD/0em6ZNS0ewJCIqTgw7ZHKZmUBYGLBwoTROW7u2NCG5dm1zV2Y+NjZPe2UKYmwvUXa2dBXURo30w42/v+WFSSKiF8WwQyZ17RowaBBw9Ki0PmYMsHQpr6ZqLGN7iW7dAtzdTX/NCiIiS8SwQyazdas0THX/vnRhqdWrpTk6ZFo2NtKN/IiIyDi8+De9sEePgHHjgH79pKDTrJl0VhCDDhERyQHDDr2QCxekcBMRIa2/9x5w6JB0qXAiIiI54DAWPRchgK+/BiZOlHp2PD2lWz507GjuyoiIiPTJvmfH398fCoXCYBk/fjwAQAiBsLAw+Pj4wMHBAW3atMG5c+fMXLVlS00FBg+W5uc8eiTd0+r0aQYdIiKSJ9mHnZiYGCQlJemW3bt3AwD6//+EkMWLF2Pp0qVYsWIFYmJi4O3tjQ4dOiDd2MvaUpHExEinO0dFSRNlFy0Cfv1VujIwERGRHMk+7Hh4eMDb21u3/Pzzz6hcuTKCgoIghMDy5cvxwQcfoE+fPqhTpw7WrVuHhw8fYtOmTeYu3aJkZwOffgoEBkpX7vX3l+bmTJ9unntbERERGatUfU09efIEGzduxIgRI6BQKBAfH4/k5GR0zDV+olQqERQUhCNHjuR7HI1Gg7S0NL2F8peSAnTrBkybJl3jpX9/6Wyr5s3NXRkREVHhSlXY2b59O+7fv4+QkBAAQHJyMgDAy8tLr52Xl5fusbwsXLgQbm5uusXPz6/Yai7t9u4F6teXhqrs7YH//Ee6GjJvQUBERKVFqQo7a9asQZcuXeDj46O3XfHM9fGFEAbbcps5cyZSU1N1S2JiYrHUW5plZQEffCBNPk5Olm71EBMDvPUWb0dARESlS6k59fzatWvYs2cPoqOjddu8/39WbHJyMlS5LimbkpJi0NuTm1KphFKpLL5iS7lr16SzrXJGAt96C1i2jLd8ICKi0qnU9OxERkbC09MT3bp1020LCAiAt7e37gwtQJrXc/DgQQQGBpqjzFIvOhpo0EAKOq6uwLffSkNXDDpERFRalYqenezsbERGRiI4OBg2Nk9LVigUCA0NRXh4OKpWrYqqVasiPDwcjo6OGDx4sBkrLn0ePQKmTHl6JeRXXpFOL+eVkImIqLQrFWFnz549SEhIwIgRIwwemz59Oh49eoRx48bh3r17aNasGXbt2gUXFxczVFo6XbgADBgAnDkjrU+fDsyfD9jamrcuIiIiU1AIIYS5izC3tLQ0uLm5ITU1Fa6uruYup8QIAURGSrd8ePhQuuXD+vVAp07mroyIiKhwxn5/l4qeHTK9tDRg7Fhg82Zp/bXXpHtb8UrIRERkaUrNBGUynZgYoGFDKehYWwMLFwI7dzLoEBGRZWLPzkskO1s6hfy996Tr6KjVUuBp0cLclRERERUfhp2XREoKEBIC7NghrfftC/z3v7wSMhERWT4OY70E9u2Trp2zY4d0y4eVK4HvvmPQISKilwPDjgXLygI+/FCafJyUBNSsCRw/DowZw1s+EBHRy4PDWBYqIUG65cPvv0vro0cDy5fzSshERPTyYdixQNu2ASNGAPfvS7d8WLVKumggERHRy4jDWBbk8WNg/HigTx8p6LzyCvDnnww6RET0cmPYsRAXLwLNmgFffSWtT5sGHDoEVKpk3rqIiIjMjcNYpZwQwNq1wIQJ0i0fPDykKyHzlg9EREQShp1SLC0NePttYNMmab19eynoqFTmrYuIiEhOOIxVSp04ATRqJAUda2sgPBzYtYtBh4iI6Fns2SllsrOlU8jfew/IzJRu+bBpExAYaO7KiIiI5IlhpxS5dUu65cP//iet9+0LrF4NlC1r1rKIiIhkjcNYpcT+/UD9+lLQUSqBiAjplg8MOkRERAVj2JG5rCxg9mxp8nHOLR9iYoCxY3nLByIiImNwGEvGEhKAIUOAw4el9VGjpPk6Tk5mLYuIiKhUYdiRqe3bpVs+3LsHuLhIt3wYONDcVREREZU+HMaSmcePgYkTgd69paDTtCkQG8ugQ0RE9LwYdmTk4kWgeXNgxQppfdo0aQiLt3wgIiJ6fhzGkgEhgHXrpJt45tzyYf16oHNnc1dGRERU+jHsmFl6unTLh2++kdZ5ywciIiLT4jCWGZ08Kd3y4Ztvnt7yYedOBh0iIiJTYs+OGQghnUI+Y4Z0y4eKFYHNm3nLByIiouLAsFPCbt0Chg8HfvlFWu/TB/jvf3klZCIiouLCYawSdOAA0KCBFHSUSuCrr4Dvv2fQISIiKk4MOyUg55YP7doBN29Kt3w4flyamMxbPhARERUvDmMVs8REYPDgp7d8GDkS+Owz3vKBiIiopDDsFKMffpDm5/CWD0RERObDsFNMhAC+/vrpLR82bwYqVzZ3VURERC8fztkpJgqFFHbCwqQhLAYdIiIi82DPTjFydwfmzDF3FURERC832ffs3LhxA0OHDoW7uzscHR3RoEEDnDx5Uvd4SEgIFAqF3tK8eXMzVkxERERyIuuenXv37qFly5Zo27YtduzYAU9PT1y5cgVlypTRa9e5c2dERkbq1u3s7Eq4UiIiIpIrWYedRYsWwc/PTy/I+Pv7G7RTKpXw9vYuwcqIiIiotJD1MNaPP/6IJk2aoH///vD09ETDhg2xevVqg3YHDhyAp6cnqlWrhtGjRyMlJaXA42o0GqSlpektREREZJlkHXb++ecfREREoGrVqti5cyfGjh2LSZMmYf369bo2Xbp0wTfffIN9+/ZhyZIliImJQbt27aDRaPI97sKFC+Hm5qZb/Pz8SuLlEBERkRkohBDC3EXkx87ODk2aNMGRI0d02yZNmoSYmBgcPXo0z32SkpKgVqsRFRWFPn365NlGo9HohaG0tDT4+fkhNTUVrq6upn0RREREVCzS0tLg5uZW6Pe3rHt2VCoVatWqpbetZs2aSEhIKHAftVqNS5cu5dtGqVTC1dVVbyEiIiLLJOuw07JlS8TFxelt+/vvv6FWq/Pd586dO0hMTIRKpSru8oiIiKgUkHXYeeedd3Ds2DGEh4fj8uXL2LRpE1atWoXx48cDADIyMjB16lQcPXoUV69exYEDB9CjRw+UL18evXv3NnP1REREJAeyDjtNmzbFtm3bsHnzZtSpUwfz5s3D8uXLMWTIEACAtbU1zpw5g549e6JatWoIDg5GtWrVcPToUbi4uJi5eiIiIpIDWU9QLinGTnAiIiIi+bCICcpEREREL4phh4iIiCwaww4RERFZNIYdIiIismgMO0RERGTRGHaIiIjIojHsEBERkUVj2CEiIiKLxrBDREREFo1hh4iIiCwaww4RERFZNIYdIiIismgMO0RERGTRGHaIiIjIojHsEBERkUVj2CEiIiKLxrBDREREFo1hh4iIiCwaww4RERFZNIYdIiIismgMO0RERGTRGHaIiIjIojHsEBERkUVj2CEiIiKLxrBDREREFs3G3AVYKq0WOHQISEoCVCqgVSvA2trcVREREb18GHaKQXQ0MHkycP36022+vsBnnwF9+pivLiIiopcRh7FMLDoa6NdPP+gAwI0b0vboaPPURURE9LJi2DEhrVbq0RHC8LGcbaGhUjsiIiIqGQw7JnTokGGPTm5CAImJUjsiIiIqGQw7JpSUZNp2RERE9OIYdkxIpTJtOyIiInpxsg87N27cwNChQ+Hu7g5HR0c0aNAAJ0+e1D0uhEBYWBh8fHzg4OCANm3a4Ny5c2aptVUr6awrhSLvxxUKwM9PakdEREQlQ9Zh5969e2jZsiVsbW2xY8cOnD9/HkuWLEGZMmV0bRYvXoylS5dixYoViImJgbe3Nzp06ID09PQSr9faWjq9HDAMPDnry5fzejtEREQlSSFEXucOycN7772H33//HYfymdErhICPjw9CQ0MxY8YMAIBGo4GXlxcWLVqEMWPGGPU8aWlpcHNzQ2pqKlxdXV+47ryus+PnJwUdXmeHiIjINIz9/pZ1z86PP/6IJk2aoH///vD09ETDhg2xevVq3ePx8fFITk5Gx44ddduUSiWCgoJw5MgRc5QMQAo0V68C+/cDmzZJf8bHM+gQERGZg6yvoPzPP/8gIiIC7777Lt5//30cP34ckyZNglKpxLBhw5CcnAwA8PLy0tvPy8sL165dy/e4Go0GGo1Gt56Wlmby2q2tgTZtTH5YIiIiKiJZh53s7Gw0adIE4eHhAICGDRvi3LlziIiIwLBhw3TtFM9MkBFCGGzLbeHChZg7d27xFE1ERESyIuthLJVKhVq1aultq1mzJhISEgAA3t7eAKDr4cmRkpJi0NuT28yZM5GamqpbEhMTTVw5ERERyYWsw07Lli0RFxent+3vv/+GWq0GAAQEBMDb2xu7d+/WPf7kyRMcPHgQgYGB+R5XqVTC1dVVbyEiIiLLJOthrHfeeQeBgYEIDw/HG2+8gePHj2PVqlVYtWoVAGn4KjQ0FOHh4ahatSqqVq2K8PBwODo6YvDgwWaunoiIiORA1mGnadOm2LZtG2bOnImPPvoIAQEBWL58OYYMGaJrM336dDx69Ajjxo3DvXv30KxZM+zatQsuLi5mrJyIiIjkQtbX2Skppr7ODhERERU/i7jODhEREdGLYtghIiIii8awQ0RERBZN1hOUS0rOtKXiuJIyERERFY+c7+3Cph8z7AC6O6T7+fmZuRIiIiIqqvT0dLi5ueX7OM/GgnRbips3b8LFxaXA20wUVVpaGvz8/JCYmMizvIzA98t4fK+Mx/fKeHyvjMf3ynjF+V4JIZCeng4fHx9YWeU/M4c9OwCsrKzg6+tbbMfnVZqLhu+X8fheGY/vlfH4XhmP75Xxiuu9KqhHJwcnKBMREZFFY9ghIiIii8awU4yUSiXmzJkDpVJp7lJKBb5fxuN7ZTy+V8bje2U8vlfGk8N7xQnKREREZNHYs0NEREQWjWGHiIiILBrDDhEREVk0hh0iIiKyaAw7xeS3335Djx494OPjA4VCge3bt5u7JFlauHAhmjZtChcXF3h6eqJXr16Ii4szd1myFBERgXr16ukuzNWiRQvs2LHD3GWVCgsXLoRCoUBoaKi5S5GlsLAwKBQKvcXb29vcZcnWjRs3MHToULi7u8PR0RENGjTAyZMnzV2W7Pj7+xt8rhQKBcaPH1/itTDsFJMHDx6gfv36WLFihblLkbWDBw9i/PjxOHbsGHbv3o2srCx07NgRDx48MHdpsuPr64uPP/4YJ06cwIkTJ9CuXTv07NkT586dM3dpshYTE4NVq1ahXr165i5F1mrXro2kpCTdcubMGXOXJEv37t1Dy5YtYWtrix07duD8+fNYsmQJypQpY+7SZCcmJkbvM7V7924AQP/+/Uu8Ft4uoph06dIFXbp0MXcZsvfrr7/qrUdGRsLT0xMnT55E69atzVSVPPXo0UNvfcGCBYiIiMCxY8dQu3ZtM1UlbxkZGRgyZAhWr16N+fPnm7scWbOxsWFvjhEWLVoEPz8/REZG6rb5+/ubryAZ8/Dw0Fv/+OOPUblyZQQFBZV4LezZIVlJTU0FAJQrV87MlcibVqtFVFQUHjx4gBYtWpi7HNkaP348unXrhtdee83cpcjepUuX4OPjg4CAAAwcOBD//POPuUuSpR9//BFNmjRB//794enpiYYNG2L16tXmLkv2njx5go0bN2LEiBEmveG2sRh2SDaEEHj33Xfx6quvok6dOuYuR5bOnDkDZ2dnKJVKjB07Ftu2bUOtWrXMXZYsRUVF4eTJk1i4cKG5S5G9Zs2aYf369di5cydWr16N5ORkBAYG4s6dO+YuTXb++ecfREREoGrVqti5cyfGjh2LSZMmYf369eYuTda2b9+O+/fvIyQkxCzPz2Esko0JEybgr7/+wuHDh81dimxVr14dsbGxuH//PrZu3Yrg4GAcPHiQgecZiYmJmDx5Mnbt2gV7e3tzlyN7uYfc69atixYtWqBy5cpYt24d3n33XTNWJj/Z2dlo0qQJwsPDAQANGzbEuXPnEBERgWHDhpm5Ovlas2YNunTpAh8fH7M8P3t2SBYmTpyIH3/8Efv374evr6+5y5EtOzs7VKlSBU2aNMHChQtRv359fPbZZ+YuS3ZOnjyJlJQUNG7cGDY2NrCxscHBgwfx+eefw8bGBlqt1twlypqTkxPq1q2LS5cumbsU2VGpVAa/XNSsWRMJCQlmqkj+rl27hj179mDUqFFmq4E9O2RWQghMnDgR27Ztw4EDBxAQEGDukkoVIQQ0Go25y5Cd9u3bG5xNNHz4cNSoUQMzZsyAtbW1mSorHTQaDS5cuIBWrVqZuxTZadmypcHlMf7++2+o1WozVSR/OSeedOvWzWw1MOwUk4yMDFy+fFm3Hh8fj9jYWJQrVw4VK1Y0Y2XyMn78eGzatAk//PADXFxckJycDABwc3ODg4ODmauTl/fffx9dunSBn58f0tPTERUVhQMHDhic0UaAi4uLwbwvJycnuLu7cz5YHqZOnYoePXqgYsWKSElJwfz585GWlobg4GBzlyY777zzDgIDAxEeHo433ngDx48fx6pVq7Bq1SpzlyZL2dnZiIyMRHBwMGxszBg5BBWL/fv3CwAGS3BwsLlLk5W83iMAIjIy0tylyc6IESOEWq0WdnZ2wsPDQ7Rv317s2rXL3GWVGkFBQWLy5MnmLkOWBgwYIFQqlbC1tRU+Pj6iT58+4ty5c+YuS7Z++uknUadOHaFUKkWNGjXEqlWrzF2SbO3cuVMAEHFxcWatQyGEEOaJWURERETFjxOUiYiIyKIx7BAREZFFY9ghIiIii8awQ0RERBaNYYeIiIgsGsMOERERWTSGHSIiIrJoDDtERAAUCgW2b99u7jKIqBgw7BCR2YWEhEChUBgsnTt3NndpRGQBeG8sIpKFzp07IzIyUm+bUqk0UzVEZEnYs0NEsqBUKuHt7a23lC1bFoA0xBQREYEuXbrAwcEBAQEB+O677/T2P3PmDNq1awcHBwe4u7vjrbfeQkZGhl6br7/+GrVr14ZSqYRKpcKECRP0Hr99+zZ69+4NR0dHVK1aFT/++KPusXv37mHIkCHw8PCAg4MDqlatahDOiEieGHaIqFSYNWsW+vbti9OnT2Po0KEYNGgQLly4AAB4+PAhOnfujLJlyyImJgbfffcd9uzZoxdmIiIiMH78eLz11ls4c+YMfvzxR1SpUkXvOebOnYs33ngDf/31F7p27YohQ4bg7t27uuc/f/48duzYgQsXLiAiIgLly5cvuTeAiJ6fWW9DSkQkhAgODhbW1tbCyclJb/noo4+EEEIAEGPHjtXbp1mzZuLtt98WQgixatUqUbZsWZGRkaF7/JdffhFWVlYiOTlZCCGEj4+P+OCDD/KtAYD48MMPdesZGRlCoVCIHTt2CCGE6NGjhxg+fLhpXjARlSjO2SEiWWjbti0iIiL0tpUrV0739xYtWug91qJFC8TGxgIALly4gPr168PJyUn3eMuWLZGdnY24uDgoFArcvHkT7du3L7CGevXq6f7u5OQEFxcXpKSkAADefvtt9O3bF6dOnULHjh3Rq1cvBAYGPtdrJaKSxbBDRLLg5ORkMKxUGIVCAQAQQuj+nlcbBwcHo45na2trsG92djYAoEuXLrh27Rp++eUX7NmzB+3bt8f48ePx6aefFqlmIip5nLNDRKXCsWPHDNZr1KgBAKhVqxZiY2Px4MED3eO///47rKysUK1aNbi4uMDf3x979+59oRo8PDwQEhKCjRs3Yvny5Vi1atULHY+ISgZ7dohIFjQaDZKTk/W22djY6CYBf/fdd2jSpAleffVVfPPNNzh+/DjWrFkDABgyZAjmzJmD4OBghIWF4datW5g4cSLefPNNeHl5AQDCwsIwduxYeHp6okuXLkhPT8fvv/+OiRMnGlXf7Nmz0bhxY9SuXRsajQY///wzatasacJ3gIiKC8MOEcnCr7/+CpVKpbetevXquHjxIgDpTKmoqCiMGzcO3t7e+Oabb1CrVi0AgKOjI3bu3InJkyejadOmcHR0RN++fbF06VLdsYKDg/H48WMsW7YMU6dORfny5dGvXz+j67Ozs8PMmTNx9epVODg4oFWrVoiKijLBKyei4qYQQghzF0FEVBCFQoFt27ahV69e5i6FiEohztkhIiIii8awQ0RERBaNc3aISPY42k5EL4I9O0RERGTRGHaIiIjIojHsEBERkUVj2CEiIiKLxrBDREREFo1hh4iIiCwaww4RERFZNIYdIiIismgMO0RERGTR/g8+TlAW9NIN4AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.clf()   # clear figure\n",
    "\n",
    "plt.plot(epochs, acc, 'bo', label='Training acc')\n",
    "plt.plot(epochs, val_acc, 'b', label='Validation acc')\n",
    "plt.title('Training and validation accuracy')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.legend()\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compute the loss & accuracy on the test set using the best available model\n",
    "\n",
    "classifier.load_state_dict(torch.load(train_state['model_filename']))\n",
    "\n",
    "classifier = classifier.to(args.device)\n",
    "dataset.class_weights = dataset.class_weights.to(args.device)\n",
    "loss_func = nn.CrossEntropyLoss(dataset.class_weights)\n",
    "\n",
    "dataset.set_split('test')\n",
    "batch_generator = generate_batches(dataset, \n",
    "                                   batch_size=args.batch_size, \n",
    "                                   device=args.device)\n",
    "running_loss = 0.\n",
    "running_acc = 0.\n",
    "classifier.eval()\n",
    "\n",
    "y_pred_list = []         # store predicted values for confusion matrix\n",
    "y_rating_list = []  # ground truth value\n",
    "\n",
    "for batch_index, batch_dict in enumerate(batch_generator):\n",
    "    # compute the output\n",
    "    y_pred =  classifier(batch_dict['x_data'],\n",
    "                         x_lengths=batch_dict['x_length'])\n",
    "\n",
    "    # store predicted values and ground truth values for calculating confusion matrix\n",
    "    y_pred_list.extend(y_pred.max(dim=1)[1].cpu().numpy())\n",
    "    y_rating_list.extend(batch_dict['y_target'].cpu().numpy())\n",
    "    \n",
    "    # compute the loss\n",
    "    loss = loss_func(y_pred, batch_dict['y_target'])\n",
    "    loss_t = loss.item()\n",
    "    running_loss += (loss_t - running_loss) / (batch_index + 1)\n",
    "\n",
    "    # compute the accuracy\n",
    "    acc_t = compute_accuracy(y_pred, batch_dict['y_target'])\n",
    "    running_acc += (acc_t - running_acc) / (batch_index + 1)\n",
    "\n",
    "train_state['test_loss'] = running_loss\n",
    "train_state['test_acc'] = running_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test loss: 0.5563440352678299;\n",
      "Test Accuracy: 70.6108870967742\n"
     ]
    }
   ],
   "source": [
    "print(\"Test loss: {};\".format(train_state['test_loss']))\n",
    "print(\"Test Accuracy: {}\".format(train_state['test_acc']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['negative', 'positive']\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
    "rating_classes = []\n",
    "for i in range(len(dataset._vectorizer.rating_vocab)):\n",
    "    rating_classes.append(dataset._vectorizer.rating_vocab.lookup_index(i))\n",
    "print(rating_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True       negative  positive\n",
      "Predicted                    \n",
      "negative        611       281\n",
      "positive        189       518\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "cm = confusion_matrix(y_rating_list, y_pred_list)\n",
    "cm_df = pd.DataFrame(cm.T, index=rating_classes, columns=rating_classes)\n",
    "cm_df.index.name = 'Predicted'\n",
    "cm_df.columns.name = 'True'\n",
    "print(cm_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.68      0.76      0.72       800\n",
      "           1       0.73      0.65      0.69       799\n",
      "\n",
      "    accuracy                           0.71      1599\n",
      "   macro avg       0.71      0.71      0.71      1599\n",
      "weighted avg       0.71      0.71      0.71      1599\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_rating_list, y_pred_list))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_sentiment(review, classifier, vectorizer):\n",
    "    vectorized_review, vec_length = vectorizer.vectorize(review)\n",
    "    vectorized_review = torch.tensor(vectorized_review).unsqueeze(dim=0)\n",
    "    vec_length = torch.tensor([vec_length], dtype=torch.int64)\n",
    "    \n",
    "    result = classifier(vectorized_review, vec_length, apply_softmax=True)\n",
    "    probability_values, indices = result.max(dim=1)\n",
    "    \n",
    "    index = indices.item()\n",
    "    prob_value = probability_values.item()\n",
    "\n",
    "    predicted_rating = vectorizer.rating_vocab.lookup_index(index)\n",
    "\n",
    "    return {'sentiment': predicted_rating, 'probability': prob_value, 'review': review}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'sentiment': 'negative', 'probability': 0.8569846749305725, 'review': 'i went here for restaurant week . the restaurant is not that nice inside the wait staff was not very professional . i ordered the chop salad . it appeared to be on leaf on lettuce chopped up no avocado as stated on menu blah . my steak was ordered medium came to table well done so it was sent back . the bread was not made fresh there tasted like a roll from fry s . dessert was okay . much better places out there for much cheaper . i won t be going back !'}\n"
     ]
    }
   ],
   "source": [
    "# review = input(\"Enter a review: \")\n",
    "review = \"i went here for restaurant week . the restaurant is not that nice inside the wait staff was not very professional . i ordered the chop salad . it appeared to be on leaf on lettuce chopped up no avocado as stated on menu blah . my steak was ordered medium came to table well done so it was sent back . the bread was not made fresh there tasted like a roll from fry s . dessert was okay . much better places out there for much cheaper . i won t be going back !\"\n",
    "classifier = classifier.to(\"cpu\")\n",
    "print(predict_sentiment(review, classifier, vectorizer))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "toc": {
   "colors": {
    "hover_highlight": "#DAA520",
    "running_highlight": "#FF0000",
    "selected_highlight": "#FFD700"
   },
   "moveMenuLeft": true,
   "nav_menu": {
    "height": "120px",
    "width": "252px"
   },
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": "5",
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}