IMF_category_embeddings_ada.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import openai\n",
    "openai.api_key = \"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\" "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Food and Non-Alcoholic Beverages',\n",
       " 'Alcoholic Beverages, Tobacco, and Narcotics',\n",
       " 'Clothing and Footwear',\n",
       " 'Accommodation, Utilities, and Services',\n",
       " 'Furnishings, household equipment and routine maintenance of the house',\n",
       " 'Health',\n",
       " 'Transportation',\n",
       " 'Communications',\n",
       " 'Recreation and Culture',\n",
       " 'Education',\n",
       " 'Restaurants and Hotels',\n",
       " 'Miscellaneous Goods and Services']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "categories = pd.read_csv('IMF_categories.csv')\n",
    "primary_categories = list(categories['Category'].unique()) \n",
    "primary_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#x = openai.Embedding.create(input=\"I want embeddings for this text\", engine=\"text-embedding-ada-002\") \n",
    "category_embeddings = openai.Embedding.create(input=primary_categories, engine=\"text-embedding-ada-002\") # create embeddings for all categories, if we pass it a list it "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "embedding categories cost 0.0000248 cents american\n"
     ]
    }
   ],
   "source": [
    "assert len(category_embeddings['data']) == len(primary_categories), \"Embeddings could not be found for all categories\" # Test\n",
    "n_tokens = int(category_embeddings['usage']['total_tokens'])\n",
    "cost_per_token = 0.0004/1000\n",
    "cost = cost_per_token * n_tokens\n",
    "cost_formatted = format(cost, '.7f')\n",
    "print(f\"embedding categories cost {cost_formatted} cents american\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_embeds2 = [category['embedding'] for category in category_embeddings['data']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "pork_embedding = openai.Embedding.create(input=\"Pork\", engine=\"text-embedding-ada-002\")['data'][0]['embedding']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The similarity between Pork and Health is 0.801\n",
      "The similarity between Pork and Food and Non-Alcoholic Beverages is 0.792\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.801, 0.792]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def compare_embeddings(phrase1, phrase2):\n",
    "    # Give it 2 phrases and it will return a measure of similarity\n",
    "    embedding1 = openai.Embedding.create(input=phrase1, engine=\"text-embedding-ada-002\")['data'][0]['embedding']\n",
    "    embedding2 = openai.Embedding.create(input=phrase2, engine=\"text-embedding-ada-002\")['data'][0]['embedding']\n",
    "    \n",
    "    cosine_dist = np.round(np.dot(embedding1,embedding2),3)\n",
    "    print(f\"The similarity between {phrase1} and {phrase2} is {cosine_dist}\")\n",
    "    return cosine_dist\n",
    "\n",
    "#TODO play around with this func and ensure it works. concepts that are similar should return higher similarity scores\n",
    "[compare_embeddings(\"Pork\",'Health'),compare_embeddings(\"Pork\",'Food and Non-Alcoholic Beverages')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The similarity between Pork and pig is 0.853\n",
      "The similarity between Pork and cow is 0.843\n",
      "The similarity between Pork and food is 0.837\n",
      "The similarity between Pork and chair is 0.787\n",
      "The similarity between Pork and tax evasion is 0.763\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.853, 0.843, 0.837, 0.787, 0.763]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[\n",
    "    compare_embeddings(\"Pork\",'pig'),\n",
    "    compare_embeddings(\"Pork\",'cow'),\n",
    "    compare_embeddings(\"Pork\",'food'),\n",
    "    compare_embeddings(\"Pork\",'chair'),\n",
    "    compare_embeddings(\"Pork\",'tax evasion')\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pork_similarity = pd.DataFrame({\"category\" : primary_categories, \"similarity\" :[np.round(np.dot(i,pork_embedding),3) for i in cat_embeds2], \"cosines\": cosine_distances})\n",
    "pork_similarity.sort_values(\"similarity\", ascending=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#TODO make this work with the code above. We need to split down the categories\n",
    "primary_cats_cleaned = primary_categories\n",
    "primary_cats_cleaned[0] = \"Food\" # rename this to a simpler category name\n",
    "primary_cats_cleaned[4] = \"Furnishings\" # rename this to a simpler category name\n",
    "primary_cats_cleaned = np.append(primary_cats_cleaned, [\"Non-Alcoholic Beverages\",\"household equipment\",\"routine house maintenance\"]) # add in more categories\n",
    "primary_cats_cleaned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#TODO do for sub categories\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}