chatbot/wiki-wizard/parse.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://github.com/facebookresearch/ParlAI/blob/master/parlai/tasks/wizard_of_wikipedia/build.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('train.json') as fopen:\n",
    "    data = json.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "informations = []\n",
    "for row in data:\n",
    "    topic = row['chosen_topic']\n",
    "    persona = row['persona']\n",
    "    for k in row['chosen_topic_passage']:\n",
    "        text = f'{topic} <> {persona} <> {k}'\n",
    "        v = {'topic': topic, 'persona': persona, 'passage': k, 'text': text}\n",
    "        informations.append(v)\n",
    "        \n",
    "len(informations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'topic': 'Science fiction',\n",
       "  'persona': 'i enjoy movies about aliens invading the earth.',\n",
       "  'passage': 'Science fiction (often shortened to SF or sci-fi) is a genre of speculative fiction, typically dealing with imaginative concepts such as futuristic science and technology, space travel, time travel, faster than light travel, parallel universes, and extraterrestrial life.',\n",
       "  'text': 'Science fiction <> i enjoy movies about aliens invading the earth. <> Science fiction (often shortened to SF or sci-fi) is a genre of speculative fiction, typically dealing with imaginative concepts such as futuristic science and technology, space travel, time travel, faster than light travel, parallel universes, and extraterrestrial life.'},\n",
       " {'topic': 'Science fiction',\n",
       "  'persona': 'i enjoy movies about aliens invading the earth.',\n",
       "  'passage': 'Science fiction often explores the potential consequences of scientific and other innovations, and has been called a \"literature of ideas\".',\n",
       "  'text': 'Science fiction <> i enjoy movies about aliens invading the earth. <> Science fiction often explores the potential consequences of scientific and other innovations, and has been called a \"literature of ideas\".'}]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "informations[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "passages = []\n",
    "for row in data:\n",
    "    topic = row['chosen_topic']\n",
    "    persona = row['persona']\n",
    "    for k in row['dialog']:\n",
    "        for z in k['retrieved_passages']:\n",
    "            for key, v in z.items():\n",
    "                for r in v:\n",
    "                    text = f'{key} <> {persona} <> {r}'\n",
    "                    v = {'topic': topic, 'persona': persona, 'text': text, 'key': key}\n",
    "                    passages.append(v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5745157"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(passages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'topic': 'Science fiction',\n",
       "  'persona': 'i enjoy movies about aliens invading the earth.',\n",
       "  'text': 'Hyperspace (science fiction) <> i enjoy movies about aliens invading the earth. <> Hyperspace is a faster-than-light (FTL) method of traveling used in science fiction.',\n",
       "  'key': 'Hyperspace (science fiction)'},\n",
       " {'topic': 'Science fiction',\n",
       "  'persona': 'i enjoy movies about aliens invading the earth.',\n",
       "  'text': 'Hyperspace (science fiction) <> i enjoy movies about aliens invading the earth. <> It is typically described as an alternative \"sub-region\" of space co-existing with our own universe which may be entered using an energy field or other device.',\n",
       "  'key': 'Hyperspace (science fiction)'}]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "passages[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "dialogs = []\n",
    "\n",
    "for row in data:\n",
    "    topic = row['chosen_topic']\n",
    "    persona = row['persona']\n",
    "    for k in range(1, len(row['dialog'])):\n",
    "        before = row['dialog'][k - 1]['text']\n",
    "        now = row['dialog'][k]['text']\n",
    "        text = f'{persona} <> {before} <> {now}'\n",
    "        v = {'topic': topic, 'persona': persona, 'before': before, 'now': now, 'text': text}\n",
    "        dialogs.append(v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "148357"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dialogs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'topic': 'Science fiction',\n",
       " 'persona': 'i enjoy movies about aliens invading the earth.',\n",
       " 'before': \"I think science fiction is an amazing genre for anything. Future science, technology, time travel, FTL travel, they're all such interesting concepts.\",\n",
       " 'now': \"I'm a huge fan of science fiction myself! \",\n",
       " 'text': \"i enjoy movies about aliens invading the earth. <> I think science fiction is an amazing genre for anything. Future science, technology, time travel, FTL travel, they're all such interesting concepts. <> I'm a huge fan of science fiction myself! \"}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dialogs[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('valid_random_split.json') as fopen:\n",
    "    valid = json.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "for row in valid:\n",
    "    topic = row['chosen_topic']\n",
    "    persona = row['persona']\n",
    "    for k in row['chosen_topic_passage']:\n",
    "        text = f'{topic} <> {persona} <> {k}'\n",
    "        v = {'topic': topic, 'persona': persona, 'passage': k, 'text': text}\n",
    "        informations.append(v)\n",
    "        \n",
    "len(informations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "for row in valid:\n",
    "    topic = row['chosen_topic']\n",
    "    persona = row['persona']\n",
    "    for k in row['dialog']:\n",
    "        for z in k['retrieved_passages']:\n",
    "            for key, v in z.items():\n",
    "                for r in v:\n",
    "                    text = f'{key} <> {persona} <> {r}'\n",
    "                    v = {'topic': topic, 'persona': persona, 'text': text, 'key': key}\n",
    "                    passages.append(v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "for row in valid:\n",
    "    topic = row['chosen_topic']\n",
    "    persona = row['persona']\n",
    "    for k in range(1, len(row['dialog'])):\n",
    "        before = row['dialog'][k - 1]['text']\n",
    "        now = row['dialog'][k]['text']\n",
    "        text = f'{persona} <> {before} <> {now}'\n",
    "        v = {'topic': topic, 'persona': persona, 'before': before, 'now': now, 'text': text}\n",
    "        dialogs.append(v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('valid_topic_split.json') as fopen:\n",
    "    valid = json.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "for row in valid:\n",
    "    topic = row['chosen_topic']\n",
    "    persona = row['persona']\n",
    "    for k in row['chosen_topic_passage']:\n",
    "        text = f'{topic} <> {persona} <> {k}'\n",
    "        v = {'topic': topic, 'persona': persona, 'passage': k, 'text': text}\n",
    "        informations.append(v)\n",
    "        \n",
    "len(informations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6359092"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for row in valid:\n",
    "    topic = row['chosen_topic']\n",
    "    persona = row['persona']\n",
    "    for k in row['dialog']:\n",
    "        for z in k['retrieved_passages']:\n",
    "            for key, v in z.items():\n",
    "                for r in v:\n",
    "                    text = f'{key} <> {persona} <> {r}'\n",
    "                    v = {'topic': topic, 'persona': persona, 'text': text, 'key': key}\n",
    "                    passages.append(v)\n",
    "                    \n",
    "len(passages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "164124"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for row in valid:\n",
    "    topic = row['chosen_topic']\n",
    "    persona = row['persona']\n",
    "    for k in range(1, len(row['dialog'])):\n",
    "        before = row['dialog'][k - 1]['text']\n",
    "        now = row['dialog'][k]['text']\n",
    "        text = f'{persona} <> {before} <> {now}'\n",
    "        v = {'topic': topic, 'persona': persona, 'before': before, 'now': now, 'text': text}\n",
    "        dialogs.append(v)\n",
    "        \n",
    "len(dialogs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('test_topic_split.json') as fopen:\n",
    "    test = json.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "201231"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for row in test:\n",
    "    topic = row['chosen_topic']\n",
    "    persona = row['persona']\n",
    "    for k in row['chosen_topic_passage']:\n",
    "        text = f'{topic} <> {persona} <> {k}'\n",
    "        v = {'topic': topic, 'persona': persona, 'passage': k, 'text': text}\n",
    "        informations.append(v)\n",
    "        \n",
    "len(informations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6658514"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for row in test:\n",
    "    topic = row['chosen_topic']\n",
    "    persona = row['persona']\n",
    "    for k in row['dialog']:\n",
    "        for z in k['retrieved_passages']:\n",
    "            for key, v in z.items():\n",
    "                for r in v:\n",
    "                    text = f'{key} <> {persona} <> {r}'\n",
    "                    v = {'topic': topic, 'persona': persona, 'text': text, 'key': key}\n",
    "                    passages.append(v)\n",
    "                    \n",
    "len(passages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "171938"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for row in test:\n",
    "    topic = row['chosen_topic']\n",
    "    persona = row['persona']\n",
    "    for k in range(1, len(row['dialog'])):\n",
    "        before = row['dialog'][k - 1]['text']\n",
    "        now = row['dialog'][k]['text']\n",
    "        text = f'{persona} <> {before} <> {now}'\n",
    "        v = {'topic': topic, 'persona': persona, 'before': before, 'now': now, 'text': text}\n",
    "        dialogs.append(v)\n",
    "        \n",
    "len(dialogs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('test_random_split.json') as fopen:\n",
    "    test = json.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "210354"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for row in test:\n",
    "    topic = row['chosen_topic']\n",
    "    persona = row['persona']\n",
    "    for k in row['chosen_topic_passage']:\n",
    "        text = f'{topic} <> {persona} <> {k}'\n",
    "        v = {'topic': topic, 'persona': persona, 'passage': k, 'text': text}\n",
    "        informations.append(v)\n",
    "        \n",
    "len(informations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6958477"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for row in test:\n",
    "    topic = row['chosen_topic']\n",
    "    persona = row['persona']\n",
    "    for k in row['dialog']:\n",
    "        for z in k['retrieved_passages']:\n",
    "            for key, v in z.items():\n",
    "                for r in v:\n",
    "                    text = f'{key} <> {persona} <> {r}'\n",
    "                    v = {'topic': topic, 'persona': persona, 'text': text, 'key': key}\n",
    "                    passages.append(v)\n",
    "                    \n",
    "len(passages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "179688"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for row in test:\n",
    "    topic = row['chosen_topic']\n",
    "    persona = row['persona']\n",
    "    for k in range(1, len(row['dialog'])):\n",
    "        before = row['dialog'][k - 1]['text']\n",
    "        now = row['dialog'][k]['text']\n",
    "        text = f'{persona} <> {before} <> {now}'\n",
    "        v = {'topic': topic, 'persona': persona, 'before': before, 'now': now, 'text': text}\n",
    "        dialogs.append(v)\n",
    "        \n",
    "len(dialogs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'topic': 'Yoga',\n",
       "  'persona': 'i am a yoga instructor.',\n",
       "  'before': 'I have not!  But I have read that laugh tracks are used sometimes?  Is that true?',\n",
       "  'now': \"I'm not sure. In my class, it's just the participants laughing our heads off. Have you heard of yoga with baby goats?\",\n",
       "  'text': \"i am a yoga instructor. <> I have not!  But I have read that laugh tracks are used sometimes?  Is that true? <> I'm not sure. In my class, it's just the participants laughing our heads off. Have you heard of yoga with baby goats?\"},\n",
       " {'topic': 'Yoga',\n",
       "  'persona': 'i am a yoga instructor.',\n",
       "  'before': \"I'm not sure. In my class, it's just the participants laughing our heads off. Have you heard of yoga with baby goats?\",\n",
       "  'now': \"haha that sounds fun.  I have heard of goat yoga...in fact a friend is trying to get me to go to a class...but I'm not sure about that!\",\n",
       "  'text': \"i am a yoga instructor. <> I'm not sure. In my class, it's just the participants laughing our heads off. Have you heard of yoga with baby goats? <> haha that sounds fun.  I have heard of goat yoga...in fact a friend is trying to get me to go to a class...but I'm not sure about that!\"},\n",
       " {'topic': 'Yoga',\n",
       "  'persona': 'i am a yoga instructor.',\n",
       "  'before': \"haha that sounds fun.  I have heard of goat yoga...in fact a friend is trying to get me to go to a class...but I'm not sure about that!\",\n",
       "  'now': \"Well, I guess it's only fair that I tell you my sister tried hot yoga and almost passed out, so be sure you're hydrated.\",\n",
       "  'text': \"i am a yoga instructor. <> haha that sounds fun.  I have heard of goat yoga...in fact a friend is trying to get me to go to a class...but I'm not sure about that! <> Well, I guess it's only fair that I tell you my sister tried hot yoga and almost passed out, so be sure you're hydrated.\"},\n",
       " {'topic': 'Romaine lettuce',\n",
       "  'persona': 'i can t stand salads.',\n",
       "  'before': 'I do not care for romaine lettuce at all.',\n",
       "  'now': 'Why but i like the fact that  it is tolerant of heat,  Unlike most lettuces',\n",
       "  'text': 'i can t stand salads. <> I do not care for romaine lettuce at all. <> Why but i like the fact that  it is tolerant of heat,  Unlike most lettuces'},\n",
       " {'topic': 'Romaine lettuce',\n",
       "  'persona': 'i can t stand salads.',\n",
       "  'before': 'Why but i like the fact that  it is tolerant of heat,  Unlike most lettuces',\n",
       "  'now': 'I just have never cared for romaine lettuce. Something about the taste. I was not aware it was tolerant of heat. Do you know anything else interesting about it?',\n",
       "  'text': 'i can t stand salads. <> Why but i like the fact that  it is tolerant of heat,  Unlike most lettuces <> I just have never cared for romaine lettuce. Something about the taste. I was not aware it was tolerant of heat. Do you know anything else interesting about it?'},\n",
       " {'topic': 'Romaine lettuce',\n",
       "  'persona': 'i can t stand salads.',\n",
       "  'before': 'I just have never cared for romaine lettuce. Something about the taste. I was not aware it was tolerant of heat. Do you know anything else interesting about it?',\n",
       "  'now': 'What about the milky fluid which gives the romaine the typically fine-bitter herb taste.',\n",
       "  'text': 'i can t stand salads. <> I just have never cared for romaine lettuce. Something about the taste. I was not aware it was tolerant of heat. Do you know anything else interesting about it? <> What about the milky fluid which gives the romaine the typically fine-bitter herb taste.'},\n",
       " {'topic': 'Romaine lettuce',\n",
       "  'persona': 'i can t stand salads.',\n",
       "  'before': 'What about the milky fluid which gives the romaine the typically fine-bitter herb taste.',\n",
       "  'now': 'I was not aware that is where it got the bitter taste but I think that is what I do not like. I do know that it is the usual lettuce used in Caesar salad. That is about all I know about romaine lettuce.',\n",
       "  'text': 'i can t stand salads. <> What about the milky fluid which gives the romaine the typically fine-bitter herb taste. <> I was not aware that is where it got the bitter taste but I think that is what I do not like. I do know that it is the usual lettuce used in Caesar salad. That is about all I know about romaine lettuce.'},\n",
       " {'topic': 'Romaine lettuce',\n",
       "  'persona': 'i can t stand salads.',\n",
       "  'before': 'I was not aware that is where it got the bitter taste but I think that is what I do not like. I do know that it is the usual lettuce used in Caesar salad. That is about all I know about romaine lettuce.',\n",
       "  'now': 'Yes that is it. You should pay attention to the  thick ribs, especially on the older outer leaves',\n",
       "  'text': 'i can t stand salads. <> I was not aware that is where it got the bitter taste but I think that is what I do not like. I do know that it is the usual lettuce used in Caesar salad. That is about all I know about romaine lettuce. <> Yes that is it. You should pay attention to the  thick ribs, especially on the older outer leaves'},\n",
       " {'topic': 'Romaine lettuce',\n",
       "  'persona': 'i can t stand salads.',\n",
       "  'before': 'Yes that is it. You should pay attention to the  thick ribs, especially on the older outer leaves',\n",
       "  'now': 'What are the thick ribs? Is that where the milky fluid is that gives it that bitter taste?',\n",
       "  'text': 'i can t stand salads. <> Yes that is it. You should pay attention to the  thick ribs, especially on the older outer leaves <> What are the thick ribs? Is that where the milky fluid is that gives it that bitter taste?'},\n",
       " {'topic': 'Romaine lettuce',\n",
       "  'persona': 'i can t stand salads.',\n",
       "  'before': 'What are the thick ribs? Is that where the milky fluid is that gives it that bitter taste?',\n",
       "  'now': 'Yes that is why it is a common salad green, and is the usual lettuce used in Caesar salad.',\n",
       "  'text': 'i can t stand salads. <> What are the thick ribs? Is that where the milky fluid is that gives it that bitter taste? <> Yes that is why it is a common salad green, and is the usual lettuce used in Caesar salad.'}]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dialogs[-10:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('dialogs.json', 'w') as fopen:\n",
    "    json.dump(dialogs, fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('passages.json', 'w') as fopen:\n",
    "    json.dump(passages, fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('informations.json', 'w') as fopen:\n",
    "    json.dump(informations, fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}