word2vec_from_scratch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import re\n",
    "from sklearn.datasets import fetch_20newsgroups\n",
    "import nltk\n",
    "from sklearn.manifold import TSNE\n",
    "from sklearn.decomposition import PCA\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes')).data\n",
    "\n",
    "regex = re.compile('[^a-zA-Zа-яА-Я ]')\n",
    "sentences = [regex.sub('', ' '.join(nltk.word_tokenize(sent.strip().lower()))).split() \n",
    "             for sent in sentences] # list of list of words\n",
    "\n",
    "sentences = [sent for sent in sentences if len(sent) > 10]\n",
    "\n",
    "np.random.shuffle(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Vocab:\n",
    "    def __init__(self):\n",
    "        self.word2index = {}\n",
    "        self.index2word = {}\n",
    "    \n",
    "    def index(self, sentences):\n",
    "        word2cnt = {}\n",
    "        for sent in sentences:\n",
    "            for word in sent:\n",
    "                if word not in self.word2index:\n",
    "                    self.word2index[word] = len(self.word2index)\n",
    "                    self.index2word[self.word2index[word]] = word\n",
    "                    word2cnt[word] = 0\n",
    "                \n",
    "                word2cnt[word] += 1\n",
    "                \n",
    "        self.distribution = np.array([word2cnt[self.index2word[i]] for i in range(len(word2cnt))])\n",
    "        self.distribution = self.distribution / self.distribution.sum()\n",
    "        self.distribution = self.distribution ** 0.75\n",
    "        self.distribution = self.distribution / self.distribution.sum()\n",
    "                    \n",
    "    def get_distribution(self):\n",
    "        return self.distribution\n",
    "                    \n",
    "    def get_size(self):\n",
    "        return len(self.word2index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "127314"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab = Vocab()\n",
    "vocab.index(sentences)\n",
    "vocab.get_size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Word2Vec:\n",
    "    def __init__(self):\n",
    "        self.sent_i = None\n",
    "        self.word_i = None\n",
    "        self.window_i = None\n",
    "        self.losses = []\n",
    "\n",
    "    def __initialize(self, window_size, vocab, embed_size):\n",
    "        if self.window_i is None:\n",
    "            print('Initializing weights')\n",
    "\n",
    "            self.vocab = vocab\n",
    "            self.center_vectors = np.random.uniform(-0.8, 0.8, (vocab.get_size(), embed_size))\n",
    "            self.context_vectors = np.random.uniform(-0.8, 0.8, (vocab.get_size(), embed_size))\n",
    "\n",
    "            self.sent_i = 0\n",
    "            self.window_i = -window_size\n",
    "            self.word_i = window_size\n",
    "\n",
    "    def __indexes_update(self, window_size, sent_size, corpus_size):\n",
    "        self.window_i += 1\n",
    "        if self.window_i == 0:\n",
    "            self.window_i = 1\n",
    "\n",
    "        if self.window_i == window_size + 1:\n",
    "            self.window_i = - window_size\n",
    "\n",
    "            self.word_i += 1\n",
    "            if self.word_i + window_size >= sent_size:\n",
    "                self.word_i = window_size\n",
    "                self.sent_i += 1\n",
    "                if self.sent_i == corpus_size:\n",
    "                    self.sent_i = 0\n",
    "\n",
    "    def train(self, vocab, sentences,\n",
    "              embed_size=100, window_size=3,\n",
    "              neg_sampling_size=15, learning_rate=0.01,\n",
    "              iters=10000, print_every=500):\n",
    "        self.__initialize(window_size, vocab, embed_size)\n",
    "\n",
    "        cur_loss = 0\n",
    "\n",
    "        i = 1\n",
    "        while i <= iters:\n",
    "            center_id = vocab.word2index[sentences[self.sent_i][self.word_i]]\n",
    "            context_id = vocab.word2index[sentences[self.sent_i][self.word_i + self.window_i]]\n",
    "\n",
    "            neg_samples_ids = np.random.choice(range(vocab.get_size()), neg_sampling_size, p=vocab.get_distribution())\n",
    "            neg_samples = self.context_vectors[neg_samples_ids]\n",
    "\n",
    "            cur_dot = np.inner(self.center_vectors[center_id], self.context_vectors[context_id])\n",
    "            dots = neg_samples.dot(self.center_vectors[center_id].reshape(-1, 1)).reshape(-1, 1)\n",
    "            sigmoids = 1 / (1 + np.exp(-dots))\n",
    "\n",
    "            # gradient calculation\n",
    "            dLdv = -self.context_vectors[context_id] * (1 - 1 / (1 + np.exp(-cur_dot))) + np.sum(neg_samples * sigmoids, axis=0)\n",
    "            dLdu = -self.center_vectors[center_id] * (1 - 1 / (1 + np.exp(-cur_dot)))\n",
    "            dLdw = neg_samples * sigmoids \n",
    "\n",
    "            # gradient step\n",
    "            self.center_vectors[center_id] -= learning_rate * dLdv\n",
    "            self.context_vectors[neg_samples_ids] -= learning_rate * dLdw\n",
    "            self.context_vectors[context_id] -= learning_rate * dLdu\n",
    "\n",
    "            self.__indexes_update(window_size, len(sentences[self.sent_i]), len(sentences))\n",
    "\n",
    "            loss = -np.log(1 + np.exp(-cur_dot)) - np.log(1 - sigmoids).sum()\n",
    "            cur_loss = cur_loss * 0.95 + loss * 0.05\n",
    "\n",
    "            if i % print_every == 0:\n",
    "                print(f'Iteration {i} loss = {round(cur_loss, 2)}')\n",
    "                self.losses.append(cur_loss)\n",
    "            i += 1\n",
    "\n",
    "    def get_vector(self, word):\n",
    "        return self.center_vectors[self.vocab.word2index[word]]\n",
    "\n",
    "    def most_closest(self, word, n=10):\n",
    "        base = self.get_vector(word).reshape(-1, 1)\n",
    "\n",
    "        distances = self.center_vectors.dot(base) / np.linalg.norm(self.center_vectors, axis=1).reshape(-1, 1) / np.linalg.norm(base)\n",
    "\n",
    "        most_closest_indexes = distances.flatten().argsort()[-n:][::-1]\n",
    "\n",
    "        return [(self.vocab.index2word[ind], distances.flatten()[ind]) for ind in most_closest_indexes]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initializing weights\n",
      "Iteration 500 loss = 12.99\n",
      "Iteration 1000 loss = 13.6\n",
      "Iteration 1500 loss = 12.25\n",
      "Iteration 2000 loss = 12.7\n",
      "Iteration 2500 loss = 11.29\n",
      "Iteration 3000 loss = 12.86\n",
      "Iteration 3500 loss = 11.95\n",
      "Iteration 4000 loss = 12.83\n",
      "Iteration 4500 loss = 12.71\n",
      "Iteration 5000 loss = 12.51\n",
      "Iteration 5500 loss = 12.82\n",
      "Iteration 6000 loss = 11.79\n",
      "Iteration 6500 loss = 12.33\n",
      "Iteration 7000 loss = 12.13\n",
      "Iteration 7500 loss = 11.37\n",
      "Iteration 8000 loss = 11.69\n",
      "Iteration 8500 loss = 11.82\n",
      "Iteration 9000 loss = 12.09\n",
      "Iteration 9500 loss = 12.15\n",
      "Iteration 10000 loss = 11.92\n",
      "Iteration 10500 loss = 11.12\n",
      "Iteration 11000 loss = 11.93\n",
      "Iteration 11500 loss = 10.74\n",
      "Iteration 12000 loss = 11.8\n",
      "Iteration 12500 loss = 11.15\n",
      "Iteration 13000 loss = 11.1\n",
      "Iteration 13500 loss = 11.89\n",
      "Iteration 14000 loss = 12.14\n",
      "Iteration 14500 loss = 11.02\n",
      "Iteration 15000 loss = 12.41\n",
      "Iteration 15500 loss = 11.53\n",
      "Iteration 16000 loss = 11.54\n",
      "Iteration 16500 loss = 12.42\n",
      "Iteration 17000 loss = 11.05\n",
      "Iteration 17500 loss = 11.2\n",
      "Iteration 18000 loss = 11.7\n",
      "Iteration 18500 loss = 11.59\n",
      "Iteration 19000 loss = 11.46\n",
      "Iteration 19500 loss = 12.33\n",
      "Iteration 20000 loss = 10.73\n",
      "Iteration 20500 loss = 11.63\n",
      "Iteration 21000 loss = 11.5\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-18-b20357533015>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      5\u001b[0m          \u001b[0mprint_every\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m500\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      6\u001b[0m          \u001b[0mwindow_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 7\u001b[1;33m          neg_sampling_size=15)\n\u001b[0m",
      "\u001b[1;32m<ipython-input-17-4906406bad93>\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(self, vocab, sentences, embed_size, window_size, neg_sampling_size, learning_rate, iters, print_every)\u001b[0m\n\u001b[0;32m     46\u001b[0m             \u001b[0mcontext_id\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvocab\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mword2index\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0msentences\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msent_i\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mword_i\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwindow_i\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     47\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 48\u001b[1;33m             \u001b[0mneg_samples_ids\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrandom\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mchoice\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvocab\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_size\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mneg_sampling_size\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mp\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mvocab\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_distribution\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     49\u001b[0m             \u001b[0mneg_samples\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontext_vectors\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mneg_samples_ids\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     50\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mmtrand.pyx\u001b[0m in \u001b[0;36mnumpy.random.mtrand.RandomState.choice\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\numpy\\core\\getlimits.py\u001b[0m in \u001b[0;36m__new__\u001b[1;34m(cls, dtype)\u001b[0m\n\u001b[0;32m    364\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    365\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__new__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 366\u001b[1;33m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    367\u001b[0m             \u001b[0mdtype\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnumeric\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    368\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "wv = Word2Vec()\n",
    "wv.train(vocab, sentences,\n",
    "         learning_rate=0.07,\n",
    "         iters=50000, \n",
    "         print_every=500, \n",
    "         window_size=3,\n",
    "         neg_sampling_size=15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "wv.most_closest('man')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "wv.most_closest('good')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}