Dataset_Cleaning_and_Merging_Suzhou.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "8f3d1c24",
   "metadata": {},
   "source": [
    "# Dataset Cleaning and Merging\n",
    "\n",
    "by Nan BAI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "90f34616",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from argparse import Namespace\n",
    "from collections import Counter\n",
    "import pickle\n",
    "import os\n",
    "import urllib\n",
    "\n",
    "args = Namespace(\n",
    "    # Data and Path information\n",
    "    save_dir = 'data_storage/',\n",
    "    image_dir = 'data_storage/images/'\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a524cca",
   "metadata": {},
   "source": [
    "## For Small Dataset of Suzhou"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b43daad6",
   "metadata": {},
   "source": [
    "### Data Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d56dd738",
   "metadata": {},
   "outputs": [],
   "source": [
    "size_df = pd.read_csv('Suzhou/data_storage/images/photos_sizes.csv',sep='\\t').rename(columns={'Unnamed: 0':'index'})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "de9077ea",
   "metadata": {},
   "source": [
    "The small data has a maximum index of 5000 during the data collection period, in the case of Suzhou, it reaches the maximum of 4230"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4e3dcaf9",
   "metadata": {},
   "outputs": [],
   "source": [
    "size_df['index'] = size_df['index'].astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3e86d6ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4230"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(size_df['index'].unique())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "434457b0",
   "metadata": {},
   "source": [
    "Only the data with downloadable marks are kept for further analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "690200cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = size_df[size_df['candownload']==1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4cea0b45",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>candownload</th>\n",
       "      <th>url_c</th>\n",
       "      <th>url_q</th>\n",
       "      <th>url_n</th>\n",
       "      <th>url_largest</th>\n",
       "      <th>owner</th>\n",
       "      <th>owner_loc</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>comments</th>\n",
       "      <th>taken</th>\n",
       "      <th>views</th>\n",
       "      <th>people</th>\n",
       "      <th>tags</th>\n",
       "      <th>lat</th>\n",
       "      <th>lon</th>\n",
       "      <th>neighbourhood</th>\n",
       "      <th>url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>51859675789</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185967578...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185967578...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185967578...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185967578...</td>\n",
       "      <td>57210421@N04</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>灭渡桥</td>\n",
       "      <td></td>\n",
       "      <td>0.0</td>\n",
       "      <td>2022-02-03 13:33:40</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[]</td>\n",
       "      <td>31.290855</td>\n",
       "      <td>120.641824</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.flickr.com/photos/wishingbone/5185...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>51859339946</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185933994...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185933994...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185933994...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185933994...</td>\n",
       "      <td>57210421@N04</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>灭渡桥</td>\n",
       "      <td></td>\n",
       "      <td>0.0</td>\n",
       "      <td>2022-02-03 13:34:07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[]</td>\n",
       "      <td>31.290855</td>\n",
       "      <td>120.641824</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.flickr.com/photos/wishingbone/5185...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>51858385807</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838580...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838580...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838580...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838580...</td>\n",
       "      <td>57210421@N04</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>灭渡桥</td>\n",
       "      <td></td>\n",
       "      <td>0.0</td>\n",
       "      <td>2022-02-03 13:34:18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[]</td>\n",
       "      <td>31.290855</td>\n",
       "      <td>120.641824</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.flickr.com/photos/wishingbone/5185...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>51859339546</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185933954...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185933954...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185933954...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185933954...</td>\n",
       "      <td>57210421@N04</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>灭渡桥</td>\n",
       "      <td></td>\n",
       "      <td>0.0</td>\n",
       "      <td>2022-02-03 13:34:52</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[]</td>\n",
       "      <td>31.290855</td>\n",
       "      <td>120.641824</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.flickr.com/photos/wishingbone/5185...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>51858385377</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838537...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838537...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838537...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838537...</td>\n",
       "      <td>57210421@N04</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>灭渡桥</td>\n",
       "      <td></td>\n",
       "      <td>0.0</td>\n",
       "      <td>2022-02-03 13:38:02</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[]</td>\n",
       "      <td>31.290855</td>\n",
       "      <td>120.641824</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.flickr.com/photos/wishingbone/5185...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4225</th>\n",
       "      <td>38397950426</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/4582/38397950426...</td>\n",
       "      <td>https://live.staticflickr.com/4582/38397950426...</td>\n",
       "      <td>https://live.staticflickr.com/4582/38397950426...</td>\n",
       "      <td>https://live.staticflickr.com/4582/38397950426...</td>\n",
       "      <td>28571579@N03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20171112_IMG_0515</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2017-11-12 19:33:08</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[]</td>\n",
       "      <td>31.320080</td>\n",
       "      <td>120.597258</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.flickr.com/photos/george8189/38397...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4226</th>\n",
       "      <td>38397948496</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/4517/38397948496...</td>\n",
       "      <td>https://live.staticflickr.com/4517/38397948496...</td>\n",
       "      <td>https://live.staticflickr.com/4517/38397948496...</td>\n",
       "      <td>https://live.staticflickr.com/4517/38397948496...</td>\n",
       "      <td>28571579@N03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20171112_IMG_0518</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2017-11-12 19:33:52</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[]</td>\n",
       "      <td>31.319936</td>\n",
       "      <td>120.597441</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.flickr.com/photos/george8189/38397...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4227</th>\n",
       "      <td>38454117381</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/4552/38454117381...</td>\n",
       "      <td>https://live.staticflickr.com/4552/38454117381...</td>\n",
       "      <td>https://live.staticflickr.com/4552/38454117381...</td>\n",
       "      <td>https://live.staticflickr.com/4552/38454117381...</td>\n",
       "      <td>28571579@N03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20171112_IMG_0531</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2017-11-12 19:40:39</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[]</td>\n",
       "      <td>31.320094</td>\n",
       "      <td>120.597633</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.flickr.com/photos/george8189/38454...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4228</th>\n",
       "      <td>38397940256</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/4576/38397940256...</td>\n",
       "      <td>https://live.staticflickr.com/4576/38397940256...</td>\n",
       "      <td>https://live.staticflickr.com/4576/38397940256...</td>\n",
       "      <td>https://live.staticflickr.com/4576/38397940256...</td>\n",
       "      <td>28571579@N03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20171112_IMG_0539</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2017-11-12 19:45:07</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[]</td>\n",
       "      <td>31.320402</td>\n",
       "      <td>120.597236</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.flickr.com/photos/george8189/38397...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4229</th>\n",
       "      <td>50150116276</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5015011627...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5015011627...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5015011627...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5015011627...</td>\n",
       "      <td>57210421@N04</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>吴中博物馆</td>\n",
       "      <td></td>\n",
       "      <td>0.0</td>\n",
       "      <td>2020-07-18 15:49:45</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>[]</td>\n",
       "      <td>31.255743</td>\n",
       "      <td>120.645525</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.flickr.com/photos/wishingbone/5015...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3137 rows × 19 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            index  candownload  \\\n",
       "8     51859675789          1.0   \n",
       "9     51859339946          1.0   \n",
       "10    51858385807          1.0   \n",
       "11    51859339546          1.0   \n",
       "12    51858385377          1.0   \n",
       "...           ...          ...   \n",
       "4225  38397950426          1.0   \n",
       "4226  38397948496          1.0   \n",
       "4227  38454117381          1.0   \n",
       "4228  38397940256          1.0   \n",
       "4229  50150116276          1.0   \n",
       "\n",
       "                                                  url_c  \\\n",
       "8     https://live.staticflickr.com/65535/5185967578...   \n",
       "9     https://live.staticflickr.com/65535/5185933994...   \n",
       "10    https://live.staticflickr.com/65535/5185838580...   \n",
       "11    https://live.staticflickr.com/65535/5185933954...   \n",
       "12    https://live.staticflickr.com/65535/5185838537...   \n",
       "...                                                 ...   \n",
       "4225  https://live.staticflickr.com/4582/38397950426...   \n",
       "4226  https://live.staticflickr.com/4517/38397948496...   \n",
       "4227  https://live.staticflickr.com/4552/38454117381...   \n",
       "4228  https://live.staticflickr.com/4576/38397940256...   \n",
       "4229  https://live.staticflickr.com/65535/5015011627...   \n",
       "\n",
       "                                                  url_q  \\\n",
       "8     https://live.staticflickr.com/65535/5185967578...   \n",
       "9     https://live.staticflickr.com/65535/5185933994...   \n",
       "10    https://live.staticflickr.com/65535/5185838580...   \n",
       "11    https://live.staticflickr.com/65535/5185933954...   \n",
       "12    https://live.staticflickr.com/65535/5185838537...   \n",
       "...                                                 ...   \n",
       "4225  https://live.staticflickr.com/4582/38397950426...   \n",
       "4226  https://live.staticflickr.com/4517/38397948496...   \n",
       "4227  https://live.staticflickr.com/4552/38454117381...   \n",
       "4228  https://live.staticflickr.com/4576/38397940256...   \n",
       "4229  https://live.staticflickr.com/65535/5015011627...   \n",
       "\n",
       "                                                  url_n  \\\n",
       "8     https://live.staticflickr.com/65535/5185967578...   \n",
       "9     https://live.staticflickr.com/65535/5185933994...   \n",
       "10    https://live.staticflickr.com/65535/5185838580...   \n",
       "11    https://live.staticflickr.com/65535/5185933954...   \n",
       "12    https://live.staticflickr.com/65535/5185838537...   \n",
       "...                                                 ...   \n",
       "4225  https://live.staticflickr.com/4582/38397950426...   \n",
       "4226  https://live.staticflickr.com/4517/38397948496...   \n",
       "4227  https://live.staticflickr.com/4552/38454117381...   \n",
       "4228  https://live.staticflickr.com/4576/38397940256...   \n",
       "4229  https://live.staticflickr.com/65535/5015011627...   \n",
       "\n",
       "                                            url_largest         owner  \\\n",
       "8     https://live.staticflickr.com/65535/5185967578...  57210421@N04   \n",
       "9     https://live.staticflickr.com/65535/5185933994...  57210421@N04   \n",
       "10    https://live.staticflickr.com/65535/5185838580...  57210421@N04   \n",
       "11    https://live.staticflickr.com/65535/5185933954...  57210421@N04   \n",
       "12    https://live.staticflickr.com/65535/5185838537...  57210421@N04   \n",
       "...                                                 ...           ...   \n",
       "4225  https://live.staticflickr.com/4582/38397950426...  28571579@N03   \n",
       "4226  https://live.staticflickr.com/4517/38397948496...  28571579@N03   \n",
       "4227  https://live.staticflickr.com/4552/38454117381...  28571579@N03   \n",
       "4228  https://live.staticflickr.com/4576/38397940256...  28571579@N03   \n",
       "4229  https://live.staticflickr.com/65535/5015011627...  57210421@N04   \n",
       "\n",
       "           owner_loc              title                      description  \\\n",
       "8     Beijing, China                灭渡桥                                    \n",
       "9     Beijing, China                灭渡桥                                    \n",
       "10    Beijing, China                灭渡桥                                    \n",
       "11    Beijing, China                灭渡桥                                    \n",
       "12    Beijing, China                灭渡桥                                    \n",
       "...              ...                ...                              ...   \n",
       "4225             NaN  20171112_IMG_0515                              NaN   \n",
       "4226             NaN  20171112_IMG_0518                              NaN   \n",
       "4227             NaN  20171112_IMG_0531                              NaN   \n",
       "4228             NaN  20171112_IMG_0539                              NaN   \n",
       "4229  Beijing, China              吴中博物馆                                    \n",
       "\n",
       "      comments                taken  views  people tags        lat  \\\n",
       "8          0.0  2022-02-03 13:33:40    0.0     0.0   []  31.290855   \n",
       "9          0.0  2022-02-03 13:34:07    0.0     0.0   []  31.290855   \n",
       "10         0.0  2022-02-03 13:34:18    0.0     0.0   []  31.290855   \n",
       "11         0.0  2022-02-03 13:34:52    0.0     0.0   []  31.290855   \n",
       "12         0.0  2022-02-03 13:38:02    2.0     0.0   []  31.290855   \n",
       "...        ...                  ...    ...     ...  ...        ...   \n",
       "4225       0.0  2017-11-12 19:33:08   11.0     0.0   []  31.320080   \n",
       "4226       0.0  2017-11-12 19:33:52   16.0     0.0   []  31.319936   \n",
       "4227       0.0  2017-11-12 19:40:39   18.0     0.0   []  31.320094   \n",
       "4228       0.0  2017-11-12 19:45:07    7.0     0.0   []  31.320402   \n",
       "4229       0.0  2020-07-18 15:49:45    3.0     0.0   []  31.255743   \n",
       "\n",
       "             lon  neighbourhood  \\\n",
       "8     120.641824            NaN   \n",
       "9     120.641824            NaN   \n",
       "10    120.641824            NaN   \n",
       "11    120.641824            NaN   \n",
       "12    120.641824            NaN   \n",
       "...          ...            ...   \n",
       "4225  120.597258            NaN   \n",
       "4226  120.597441            NaN   \n",
       "4227  120.597633            NaN   \n",
       "4228  120.597236            NaN   \n",
       "4229  120.645525            NaN   \n",
       "\n",
       "                                                    url  \n",
       "8     https://www.flickr.com/photos/wishingbone/5185...  \n",
       "9     https://www.flickr.com/photos/wishingbone/5185...  \n",
       "10    https://www.flickr.com/photos/wishingbone/5185...  \n",
       "11    https://www.flickr.com/photos/wishingbone/5185...  \n",
       "12    https://www.flickr.com/photos/wishingbone/5185...  \n",
       "...                                                 ...  \n",
       "4225  https://www.flickr.com/photos/george8189/38397...  \n",
       "4226  https://www.flickr.com/photos/george8189/38397...  \n",
       "4227  https://www.flickr.com/photos/george8189/38454...  \n",
       "4228  https://www.flickr.com/photos/george8189/38397...  \n",
       "4229  https://www.flickr.com/photos/wishingbone/5015...  \n",
       "\n",
       "[3137 rows x 19 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "15e6dd6f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://www.flickr.com/photos/wishingbone/51859675789/'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[0].url"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0709c83e",
   "metadata": {},
   "source": [
    "### Data Cleaning and Processing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c53882d1",
   "metadata": {},
   "source": [
    "#### Finding Locals and Neighbourhoods"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a06e395b",
   "metadata": {},
   "source": [
    "The data are distributed in NaN different neighbourhoods in Suzhou (which is missing from the dataset), posted by 95 different owners, among which 5 are people based in mainland of China, due to their indicated locations in Flickr user profile, who contributed 450 posts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b2e6ec56",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['neighbourhood'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4b07581e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df['neighbourhood'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d97b5442",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Beijing, China', nan, 'Valencia, España', 'Singapore, Singapore',\n",
       "       'Gif-sur-Yvette, France', '台中, 台灣', 'Boston, US', 'France',\n",
       "       'San Jose, CA., USA', 'Hong Kong', 'Vancouver BC, Canada',\n",
       "       'Sydney, Australia', 'San Francisco, America',\n",
       "       'Guandong Chunshan 中山 , China 中國 ', 'Urdorf, Switzerland',\n",
       "       'Kokomo, IN, USA', 'Scotts Valley, USA', 'Paris, France',\n",
       "       'Shanghai, China', 'Australia', 'Birmingham, UK',\n",
       "       'Hong Kong, Hong Kong', 'Brussels, Belgium', 'Milan',\n",
       "       'Nanaimo, Canada', 'Tuxedo Park, USA'], dtype=object)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['owner_loc'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2cfd4280",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Beijing, China', 'Guandong Chunshan 中山 , China 中國 ', 'Shanghai, China']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "locations = [now for now in list(df['owner_loc'].unique()) if 'china' in str(now).lower()]\n",
    "locations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2db2a3f0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zzbn1\\AppData\\Local\\Temp\\ipykernel_26468\\1482302024.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['local'] = df['owner_loc'].apply(lambda x: x in locations)\n"
     ]
    }
   ],
   "source": [
    "df['local'] = df['owner_loc'].apply(lambda x: x in locations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "71a41b4b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df[df['local']]['owner'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a78481a6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "450"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df[df['local']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e5bccc6a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "95"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df['owner'].unique())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a2b78007",
   "metadata": {},
   "source": [
    "#### Processing Tags and Converting to lists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "9b79bc6a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "re.findall(r'_content\\': \\'([a-zA-Z]+)',df['tags'].iloc[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "6ea1f304",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zzbn1\\AppData\\Local\\Temp\\ipykernel_26468\\1574953216.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['tag_list'] = df['tags'].apply(lambda x: re.findall(r'_content\\': \\'([a-zA-Z]+)',x))\n"
     ]
    }
   ],
   "source": [
    "df['tag_list'] = df['tags'].apply(lambda x: re.findall(r'_content\\': \\'([a-zA-Z]+)',x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "dbc86e3e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zzbn1\\AppData\\Local\\Temp\\ipykernel_26468\\1455675285.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['tag_len'] = df['tag_list'].apply(lambda x: len(x))\n"
     ]
    }
   ],
   "source": [
    "df['tag_len'] = df['tag_list'].apply(lambda x: len(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98e04deb",
   "metadata": {},
   "source": [
    "#### Filtering out Erotic images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "5981d8ea",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zzbn1\\AppData\\Local\\Temp\\ipykernel_26468\\4184844319.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['NOT'] = df['tag_list'].apply(lambda x: 'erotic' in x or 'nude' in x)\n"
     ]
    }
   ],
   "source": [
    "df['NOT'] = df['tag_list'].apply(lambda x: 'erotic' in x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "fdbf7095",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[~df['NOT']]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e9292194",
   "metadata": {},
   "source": [
    "#### Generating Timestamps for the posts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "248e8288",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['year'] = df['taken'].apply(lambda x: int(x[:4]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "2fb1dfd9",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['time'] = df['taken'].apply(pd.Timestamp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "f4b47b2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['week'] = df['time'].apply(lambda x: x.week)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "7e096341",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['unique_week'] = df['year'] * 100 + df['week']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "fbd2f274",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "117"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df['unique_week'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "cfe88ec1",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2022, 2021, 2010, 2015, 2007, 2019, 2012, 2020, 2014, 2013, 2018,\n",
       "       2016, 2017], dtype=int64)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['year'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "c1472daf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0,  6,  1,  4,  2,  5, 28, 25,  3, 18, 26, 22,  8,  9, 24,  7, 11,\n",
       "       14, 16, 17, 13, 10, 32, 27, 20, 33, 34, 12, 15], dtype=int64)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['tag_len'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "7551a717",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>candownload</th>\n",
       "      <th>url_c</th>\n",
       "      <th>url_q</th>\n",
       "      <th>url_n</th>\n",
       "      <th>url_largest</th>\n",
       "      <th>owner</th>\n",
       "      <th>owner_loc</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>...</th>\n",
       "      <th>neighbourhood</th>\n",
       "      <th>url</th>\n",
       "      <th>local</th>\n",
       "      <th>tag_list</th>\n",
       "      <th>tag_len</th>\n",
       "      <th>NOT</th>\n",
       "      <th>year</th>\n",
       "      <th>time</th>\n",
       "      <th>week</th>\n",
       "      <th>unique_week</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>51859675789</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185967578...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185967578...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185967578...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185967578...</td>\n",
       "      <td>57210421@N04</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>灭渡桥</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.flickr.com/photos/wishingbone/5185...</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>2022</td>\n",
       "      <td>2022-02-03 13:33:40</td>\n",
       "      <td>5</td>\n",
       "      <td>202205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>51859339946</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185933994...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185933994...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185933994...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185933994...</td>\n",
       "      <td>57210421@N04</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>灭渡桥</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.flickr.com/photos/wishingbone/5185...</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>2022</td>\n",
       "      <td>2022-02-03 13:34:07</td>\n",
       "      <td>5</td>\n",
       "      <td>202205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>51858385807</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838580...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838580...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838580...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838580...</td>\n",
       "      <td>57210421@N04</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>灭渡桥</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.flickr.com/photos/wishingbone/5185...</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>2022</td>\n",
       "      <td>2022-02-03 13:34:18</td>\n",
       "      <td>5</td>\n",
       "      <td>202205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>51859339546</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185933954...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185933954...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185933954...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185933954...</td>\n",
       "      <td>57210421@N04</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>灭渡桥</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.flickr.com/photos/wishingbone/5185...</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>2022</td>\n",
       "      <td>2022-02-03 13:34:52</td>\n",
       "      <td>5</td>\n",
       "      <td>202205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>51858385377</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838537...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838537...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838537...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838537...</td>\n",
       "      <td>57210421@N04</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>灭渡桥</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.flickr.com/photos/wishingbone/5185...</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>2022</td>\n",
       "      <td>2022-02-03 13:38:02</td>\n",
       "      <td>5</td>\n",
       "      <td>202205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4225</th>\n",
       "      <td>38397950426</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/4582/38397950426...</td>\n",
       "      <td>https://live.staticflickr.com/4582/38397950426...</td>\n",
       "      <td>https://live.staticflickr.com/4582/38397950426...</td>\n",
       "      <td>https://live.staticflickr.com/4582/38397950426...</td>\n",
       "      <td>28571579@N03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20171112_IMG_0515</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.flickr.com/photos/george8189/38397...</td>\n",
       "      <td>False</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>2017</td>\n",
       "      <td>2017-11-12 19:33:08</td>\n",
       "      <td>45</td>\n",
       "      <td>201745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4226</th>\n",
       "      <td>38397948496</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/4517/38397948496...</td>\n",
       "      <td>https://live.staticflickr.com/4517/38397948496...</td>\n",
       "      <td>https://live.staticflickr.com/4517/38397948496...</td>\n",
       "      <td>https://live.staticflickr.com/4517/38397948496...</td>\n",
       "      <td>28571579@N03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20171112_IMG_0518</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.flickr.com/photos/george8189/38397...</td>\n",
       "      <td>False</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>2017</td>\n",
       "      <td>2017-11-12 19:33:52</td>\n",
       "      <td>45</td>\n",
       "      <td>201745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4227</th>\n",
       "      <td>38454117381</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/4552/38454117381...</td>\n",
       "      <td>https://live.staticflickr.com/4552/38454117381...</td>\n",
       "      <td>https://live.staticflickr.com/4552/38454117381...</td>\n",
       "      <td>https://live.staticflickr.com/4552/38454117381...</td>\n",
       "      <td>28571579@N03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20171112_IMG_0531</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.flickr.com/photos/george8189/38454...</td>\n",
       "      <td>False</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>2017</td>\n",
       "      <td>2017-11-12 19:40:39</td>\n",
       "      <td>45</td>\n",
       "      <td>201745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4228</th>\n",
       "      <td>38397940256</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/4576/38397940256...</td>\n",
       "      <td>https://live.staticflickr.com/4576/38397940256...</td>\n",
       "      <td>https://live.staticflickr.com/4576/38397940256...</td>\n",
       "      <td>https://live.staticflickr.com/4576/38397940256...</td>\n",
       "      <td>28571579@N03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20171112_IMG_0539</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.flickr.com/photos/george8189/38397...</td>\n",
       "      <td>False</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>2017</td>\n",
       "      <td>2017-11-12 19:45:07</td>\n",
       "      <td>45</td>\n",
       "      <td>201745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4229</th>\n",
       "      <td>50150116276</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5015011627...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5015011627...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5015011627...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5015011627...</td>\n",
       "      <td>57210421@N04</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>吴中博物馆</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.flickr.com/photos/wishingbone/5015...</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>2020</td>\n",
       "      <td>2020-07-18 15:49:45</td>\n",
       "      <td>29</td>\n",
       "      <td>202029</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3137 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            index  candownload  \\\n",
       "8     51859675789          1.0   \n",
       "9     51859339946          1.0   \n",
       "10    51858385807          1.0   \n",
       "11    51859339546          1.0   \n",
       "12    51858385377          1.0   \n",
       "...           ...          ...   \n",
       "4225  38397950426          1.0   \n",
       "4226  38397948496          1.0   \n",
       "4227  38454117381          1.0   \n",
       "4228  38397940256          1.0   \n",
       "4229  50150116276          1.0   \n",
       "\n",
       "                                                  url_c  \\\n",
       "8     https://live.staticflickr.com/65535/5185967578...   \n",
       "9     https://live.staticflickr.com/65535/5185933994...   \n",
       "10    https://live.staticflickr.com/65535/5185838580...   \n",
       "11    https://live.staticflickr.com/65535/5185933954...   \n",
       "12    https://live.staticflickr.com/65535/5185838537...   \n",
       "...                                                 ...   \n",
       "4225  https://live.staticflickr.com/4582/38397950426...   \n",
       "4226  https://live.staticflickr.com/4517/38397948496...   \n",
       "4227  https://live.staticflickr.com/4552/38454117381...   \n",
       "4228  https://live.staticflickr.com/4576/38397940256...   \n",
       "4229  https://live.staticflickr.com/65535/5015011627...   \n",
       "\n",
       "                                                  url_q  \\\n",
       "8     https://live.staticflickr.com/65535/5185967578...   \n",
       "9     https://live.staticflickr.com/65535/5185933994...   \n",
       "10    https://live.staticflickr.com/65535/5185838580...   \n",
       "11    https://live.staticflickr.com/65535/5185933954...   \n",
       "12    https://live.staticflickr.com/65535/5185838537...   \n",
       "...                                                 ...   \n",
       "4225  https://live.staticflickr.com/4582/38397950426...   \n",
       "4226  https://live.staticflickr.com/4517/38397948496...   \n",
       "4227  https://live.staticflickr.com/4552/38454117381...   \n",
       "4228  https://live.staticflickr.com/4576/38397940256...   \n",
       "4229  https://live.staticflickr.com/65535/5015011627...   \n",
       "\n",
       "                                                  url_n  \\\n",
       "8     https://live.staticflickr.com/65535/5185967578...   \n",
       "9     https://live.staticflickr.com/65535/5185933994...   \n",
       "10    https://live.staticflickr.com/65535/5185838580...   \n",
       "11    https://live.staticflickr.com/65535/5185933954...   \n",
       "12    https://live.staticflickr.com/65535/5185838537...   \n",
       "...                                                 ...   \n",
       "4225  https://live.staticflickr.com/4582/38397950426...   \n",
       "4226  https://live.staticflickr.com/4517/38397948496...   \n",
       "4227  https://live.staticflickr.com/4552/38454117381...   \n",
       "4228  https://live.staticflickr.com/4576/38397940256...   \n",
       "4229  https://live.staticflickr.com/65535/5015011627...   \n",
       "\n",
       "                                            url_largest         owner  \\\n",
       "8     https://live.staticflickr.com/65535/5185967578...  57210421@N04   \n",
       "9     https://live.staticflickr.com/65535/5185933994...  57210421@N04   \n",
       "10    https://live.staticflickr.com/65535/5185838580...  57210421@N04   \n",
       "11    https://live.staticflickr.com/65535/5185933954...  57210421@N04   \n",
       "12    https://live.staticflickr.com/65535/5185838537...  57210421@N04   \n",
       "...                                                 ...           ...   \n",
       "4225  https://live.staticflickr.com/4582/38397950426...  28571579@N03   \n",
       "4226  https://live.staticflickr.com/4517/38397948496...  28571579@N03   \n",
       "4227  https://live.staticflickr.com/4552/38454117381...  28571579@N03   \n",
       "4228  https://live.staticflickr.com/4576/38397940256...  28571579@N03   \n",
       "4229  https://live.staticflickr.com/65535/5015011627...  57210421@N04   \n",
       "\n",
       "           owner_loc              title                      description  ...  \\\n",
       "8     Beijing, China                灭渡桥                                   ...   \n",
       "9     Beijing, China                灭渡桥                                   ...   \n",
       "10    Beijing, China                灭渡桥                                   ...   \n",
       "11    Beijing, China                灭渡桥                                   ...   \n",
       "12    Beijing, China                灭渡桥                                   ...   \n",
       "...              ...                ...                              ...  ...   \n",
       "4225             NaN  20171112_IMG_0515                              NaN  ...   \n",
       "4226             NaN  20171112_IMG_0518                              NaN  ...   \n",
       "4227             NaN  20171112_IMG_0531                              NaN  ...   \n",
       "4228             NaN  20171112_IMG_0539                              NaN  ...   \n",
       "4229  Beijing, China              吴中博物馆                                   ...   \n",
       "\n",
       "      neighbourhood                                                url  local  \\\n",
       "8               NaN  https://www.flickr.com/photos/wishingbone/5185...   True   \n",
       "9               NaN  https://www.flickr.com/photos/wishingbone/5185...   True   \n",
       "10              NaN  https://www.flickr.com/photos/wishingbone/5185...   True   \n",
       "11              NaN  https://www.flickr.com/photos/wishingbone/5185...   True   \n",
       "12              NaN  https://www.flickr.com/photos/wishingbone/5185...   True   \n",
       "...             ...                                                ...    ...   \n",
       "4225            NaN  https://www.flickr.com/photos/george8189/38397...  False   \n",
       "4226            NaN  https://www.flickr.com/photos/george8189/38397...  False   \n",
       "4227            NaN  https://www.flickr.com/photos/george8189/38454...  False   \n",
       "4228            NaN  https://www.flickr.com/photos/george8189/38397...  False   \n",
       "4229            NaN  https://www.flickr.com/photos/wishingbone/5015...   True   \n",
       "\n",
       "      tag_list tag_len    NOT  year                time week  unique_week  \n",
       "8           []       0  False  2022 2022-02-03 13:33:40    5       202205  \n",
       "9           []       0  False  2022 2022-02-03 13:34:07    5       202205  \n",
       "10          []       0  False  2022 2022-02-03 13:34:18    5       202205  \n",
       "11          []       0  False  2022 2022-02-03 13:34:52    5       202205  \n",
       "12          []       0  False  2022 2022-02-03 13:38:02    5       202205  \n",
       "...        ...     ...    ...   ...                 ...  ...          ...  \n",
       "4225        []       0  False  2017 2017-11-12 19:33:08   45       201745  \n",
       "4226        []       0  False  2017 2017-11-12 19:33:52   45       201745  \n",
       "4227        []       0  False  2017 2017-11-12 19:40:39   45       201745  \n",
       "4228        []       0  False  2017 2017-11-12 19:45:07   45       201745  \n",
       "4229        []       0  False  2020 2020-07-18 15:49:45   29       202029  \n",
       "\n",
       "[3137 rows x 27 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0fa2fa34",
   "metadata": {},
   "source": [
    "### Creating a New Dataset of data with text descriptions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c8d5214f",
   "metadata": {},
   "source": [
    "#### Filtering out an initial subset with text descriptions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4dc5ee07",
   "metadata": {},
   "source": [
    "In total, 2801 out of 2951 complete posts have at least one text field (description, title, or tags) that is not empty."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "e72cd6da",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>candownload</th>\n",
       "      <th>url_c</th>\n",
       "      <th>url_q</th>\n",
       "      <th>url_n</th>\n",
       "      <th>url_largest</th>\n",
       "      <th>owner</th>\n",
       "      <th>owner_loc</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>...</th>\n",
       "      <th>neighbourhood</th>\n",
       "      <th>url</th>\n",
       "      <th>local</th>\n",
       "      <th>tag_list</th>\n",
       "      <th>tag_len</th>\n",
       "      <th>NOT</th>\n",
       "      <th>year</th>\n",
       "      <th>time</th>\n",
       "      <th>week</th>\n",
       "      <th>unique_week</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>51859675789</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185967578...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185967578...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185967578...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185967578...</td>\n",
       "      <td>57210421@N04</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>灭渡桥</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.flickr.com/photos/wishingbone/5185...</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>2022</td>\n",
       "      <td>2022-02-03 13:33:40</td>\n",
       "      <td>5</td>\n",
       "      <td>202205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>51859339946</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185933994...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185933994...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185933994...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185933994...</td>\n",
       "      <td>57210421@N04</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>灭渡桥</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.flickr.com/photos/wishingbone/5185...</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>2022</td>\n",
       "      <td>2022-02-03 13:34:07</td>\n",
       "      <td>5</td>\n",
       "      <td>202205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>51858385807</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838580...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838580...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838580...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838580...</td>\n",
       "      <td>57210421@N04</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>灭渡桥</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.flickr.com/photos/wishingbone/5185...</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>2022</td>\n",
       "      <td>2022-02-03 13:34:18</td>\n",
       "      <td>5</td>\n",
       "      <td>202205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>51859339546</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185933954...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185933954...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185933954...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185933954...</td>\n",
       "      <td>57210421@N04</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>灭渡桥</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.flickr.com/photos/wishingbone/5185...</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>2022</td>\n",
       "      <td>2022-02-03 13:34:52</td>\n",
       "      <td>5</td>\n",
       "      <td>202205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>51858385377</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838537...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838537...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838537...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838537...</td>\n",
       "      <td>57210421@N04</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>灭渡桥</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.flickr.com/photos/wishingbone/5185...</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>2022</td>\n",
       "      <td>2022-02-03 13:38:02</td>\n",
       "      <td>5</td>\n",
       "      <td>202205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4225</th>\n",
       "      <td>38397950426</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/4582/38397950426...</td>\n",
       "      <td>https://live.staticflickr.com/4582/38397950426...</td>\n",
       "      <td>https://live.staticflickr.com/4582/38397950426...</td>\n",
       "      <td>https://live.staticflickr.com/4582/38397950426...</td>\n",
       "      <td>28571579@N03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20171112_IMG_0515</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.flickr.com/photos/george8189/38397...</td>\n",
       "      <td>False</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>2017</td>\n",
       "      <td>2017-11-12 19:33:08</td>\n",
       "      <td>45</td>\n",
       "      <td>201745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4226</th>\n",
       "      <td>38397948496</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/4517/38397948496...</td>\n",
       "      <td>https://live.staticflickr.com/4517/38397948496...</td>\n",
       "      <td>https://live.staticflickr.com/4517/38397948496...</td>\n",
       "      <td>https://live.staticflickr.com/4517/38397948496...</td>\n",
       "      <td>28571579@N03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20171112_IMG_0518</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.flickr.com/photos/george8189/38397...</td>\n",
       "      <td>False</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>2017</td>\n",
       "      <td>2017-11-12 19:33:52</td>\n",
       "      <td>45</td>\n",
       "      <td>201745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4227</th>\n",
       "      <td>38454117381</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/4552/38454117381...</td>\n",
       "      <td>https://live.staticflickr.com/4552/38454117381...</td>\n",
       "      <td>https://live.staticflickr.com/4552/38454117381...</td>\n",
       "      <td>https://live.staticflickr.com/4552/38454117381...</td>\n",
       "      <td>28571579@N03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20171112_IMG_0531</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.flickr.com/photos/george8189/38454...</td>\n",
       "      <td>False</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>2017</td>\n",
       "      <td>2017-11-12 19:40:39</td>\n",
       "      <td>45</td>\n",
       "      <td>201745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4228</th>\n",
       "      <td>38397940256</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/4576/38397940256...</td>\n",
       "      <td>https://live.staticflickr.com/4576/38397940256...</td>\n",
       "      <td>https://live.staticflickr.com/4576/38397940256...</td>\n",
       "      <td>https://live.staticflickr.com/4576/38397940256...</td>\n",
       "      <td>28571579@N03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20171112_IMG_0539</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.flickr.com/photos/george8189/38397...</td>\n",
       "      <td>False</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>2017</td>\n",
       "      <td>2017-11-12 19:45:07</td>\n",
       "      <td>45</td>\n",
       "      <td>201745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4229</th>\n",
       "      <td>50150116276</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5015011627...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5015011627...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5015011627...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5015011627...</td>\n",
       "      <td>57210421@N04</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>吴中博物馆</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.flickr.com/photos/wishingbone/5015...</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>2020</td>\n",
       "      <td>2020-07-18 15:49:45</td>\n",
       "      <td>29</td>\n",
       "      <td>202029</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2692 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            index  candownload  \\\n",
       "8     51859675789          1.0   \n",
       "9     51859339946          1.0   \n",
       "10    51858385807          1.0   \n",
       "11    51859339546          1.0   \n",
       "12    51858385377          1.0   \n",
       "...           ...          ...   \n",
       "4225  38397950426          1.0   \n",
       "4226  38397948496          1.0   \n",
       "4227  38454117381          1.0   \n",
       "4228  38397940256          1.0   \n",
       "4229  50150116276          1.0   \n",
       "\n",
       "                                                  url_c  \\\n",
       "8     https://live.staticflickr.com/65535/5185967578...   \n",
       "9     https://live.staticflickr.com/65535/5185933994...   \n",
       "10    https://live.staticflickr.com/65535/5185838580...   \n",
       "11    https://live.staticflickr.com/65535/5185933954...   \n",
       "12    https://live.staticflickr.com/65535/5185838537...   \n",
       "...                                                 ...   \n",
       "4225  https://live.staticflickr.com/4582/38397950426...   \n",
       "4226  https://live.staticflickr.com/4517/38397948496...   \n",
       "4227  https://live.staticflickr.com/4552/38454117381...   \n",
       "4228  https://live.staticflickr.com/4576/38397940256...   \n",
       "4229  https://live.staticflickr.com/65535/5015011627...   \n",
       "\n",
       "                                                  url_q  \\\n",
       "8     https://live.staticflickr.com/65535/5185967578...   \n",
       "9     https://live.staticflickr.com/65535/5185933994...   \n",
       "10    https://live.staticflickr.com/65535/5185838580...   \n",
       "11    https://live.staticflickr.com/65535/5185933954...   \n",
       "12    https://live.staticflickr.com/65535/5185838537...   \n",
       "...                                                 ...   \n",
       "4225  https://live.staticflickr.com/4582/38397950426...   \n",
       "4226  https://live.staticflickr.com/4517/38397948496...   \n",
       "4227  https://live.staticflickr.com/4552/38454117381...   \n",
       "4228  https://live.staticflickr.com/4576/38397940256...   \n",
       "4229  https://live.staticflickr.com/65535/5015011627...   \n",
       "\n",
       "                                                  url_n  \\\n",
       "8     https://live.staticflickr.com/65535/5185967578...   \n",
       "9     https://live.staticflickr.com/65535/5185933994...   \n",
       "10    https://live.staticflickr.com/65535/5185838580...   \n",
       "11    https://live.staticflickr.com/65535/5185933954...   \n",
       "12    https://live.staticflickr.com/65535/5185838537...   \n",
       "...                                                 ...   \n",
       "4225  https://live.staticflickr.com/4582/38397950426...   \n",
       "4226  https://live.staticflickr.com/4517/38397948496...   \n",
       "4227  https://live.staticflickr.com/4552/38454117381...   \n",
       "4228  https://live.staticflickr.com/4576/38397940256...   \n",
       "4229  https://live.staticflickr.com/65535/5015011627...   \n",
       "\n",
       "                                            url_largest         owner  \\\n",
       "8     https://live.staticflickr.com/65535/5185967578...  57210421@N04   \n",
       "9     https://live.staticflickr.com/65535/5185933994...  57210421@N04   \n",
       "10    https://live.staticflickr.com/65535/5185838580...  57210421@N04   \n",
       "11    https://live.staticflickr.com/65535/5185933954...  57210421@N04   \n",
       "12    https://live.staticflickr.com/65535/5185838537...  57210421@N04   \n",
       "...                                                 ...           ...   \n",
       "4225  https://live.staticflickr.com/4582/38397950426...  28571579@N03   \n",
       "4226  https://live.staticflickr.com/4517/38397948496...  28571579@N03   \n",
       "4227  https://live.staticflickr.com/4552/38454117381...  28571579@N03   \n",
       "4228  https://live.staticflickr.com/4576/38397940256...  28571579@N03   \n",
       "4229  https://live.staticflickr.com/65535/5015011627...  57210421@N04   \n",
       "\n",
       "           owner_loc              title                      description  ...  \\\n",
       "8     Beijing, China                灭渡桥                                   ...   \n",
       "9     Beijing, China                灭渡桥                                   ...   \n",
       "10    Beijing, China                灭渡桥                                   ...   \n",
       "11    Beijing, China                灭渡桥                                   ...   \n",
       "12    Beijing, China                灭渡桥                                   ...   \n",
       "...              ...                ...                              ...  ...   \n",
       "4225             NaN  20171112_IMG_0515                              NaN  ...   \n",
       "4226             NaN  20171112_IMG_0518                              NaN  ...   \n",
       "4227             NaN  20171112_IMG_0531                              NaN  ...   \n",
       "4228             NaN  20171112_IMG_0539                              NaN  ...   \n",
       "4229  Beijing, China              吴中博物馆                                   ...   \n",
       "\n",
       "      neighbourhood                                                url  local  \\\n",
       "8               NaN  https://www.flickr.com/photos/wishingbone/5185...   True   \n",
       "9               NaN  https://www.flickr.com/photos/wishingbone/5185...   True   \n",
       "10              NaN  https://www.flickr.com/photos/wishingbone/5185...   True   \n",
       "11              NaN  https://www.flickr.com/photos/wishingbone/5185...   True   \n",
       "12              NaN  https://www.flickr.com/photos/wishingbone/5185...   True   \n",
       "...             ...                                                ...    ...   \n",
       "4225            NaN  https://www.flickr.com/photos/george8189/38397...  False   \n",
       "4226            NaN  https://www.flickr.com/photos/george8189/38397...  False   \n",
       "4227            NaN  https://www.flickr.com/photos/george8189/38454...  False   \n",
       "4228            NaN  https://www.flickr.com/photos/george8189/38397...  False   \n",
       "4229            NaN  https://www.flickr.com/photos/wishingbone/5015...   True   \n",
       "\n",
       "      tag_list tag_len    NOT  year                time week  unique_week  \n",
       "8           []       0  False  2022 2022-02-03 13:33:40    5       202205  \n",
       "9           []       0  False  2022 2022-02-03 13:34:07    5       202205  \n",
       "10          []       0  False  2022 2022-02-03 13:34:18    5       202205  \n",
       "11          []       0  False  2022 2022-02-03 13:34:52    5       202205  \n",
       "12          []       0  False  2022 2022-02-03 13:38:02    5       202205  \n",
       "...        ...     ...    ...   ...                 ...  ...          ...  \n",
       "4225        []       0  False  2017 2017-11-12 19:33:08   45       201745  \n",
       "4226        []       0  False  2017 2017-11-12 19:33:52   45       201745  \n",
       "4227        []       0  False  2017 2017-11-12 19:40:39   45       201745  \n",
       "4228        []       0  False  2017 2017-11-12 19:45:07   45       201745  \n",
       "4229        []       0  False  2020 2020-07-18 15:49:45   29       202029  \n",
       "\n",
       "[2692 rows x 27 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_new = df[~((df['description'].isnull())&(df['title'].isnull())&(df['tag_len']==0))]\n",
    "df_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "8f7c234f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zzbn1\\AppData\\Local\\Temp\\ipykernel_26468\\2513102529.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_new['description'] = df_new['description'].fillna('')\n",
      "C:\\Users\\zzbn1\\AppData\\Local\\Temp\\ipykernel_26468\\2513102529.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_new['title'] = df_new['title'].fillna('')\n"
     ]
    }
   ],
   "source": [
    "df_new['description'] = df_new['description'].fillna('')\n",
    "df_new['title'] = df_new['title'].fillna('')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f11acc7b",
   "metadata": {},
   "source": [
    "Merging the three types of text data into one complete paragraph with the order of description -> title -> tags."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "6e133e70",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zzbn1\\AppData\\Local\\Temp\\ipykernel_26468\\475771120.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_new['text_data'] = df_new.apply(lambda x: str(x['description']) + '. It has title of ' + str(x['title']) + '. It has tags of ' + ', '.join(x['tag_list']), axis=1)\n"
     ]
    }
   ],
   "source": [
    "df_new['text_data'] = df_new.apply(lambda x: str(x['description']) + '. It has title of ' + str(x['title']) + '. It has tags of ' + ', '.join(x['tag_list']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "d68b48ff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8                                      . It has title ...\n",
       "9                                      . It has title ...\n",
       "10                                     . It has title ...\n",
       "11                                     . It has title ...\n",
       "12                                     . It has title ...\n",
       "                              ...                        \n",
       "4225    . It has title of 20171112_IMG_0515. It has ta...\n",
       "4226    . It has title of 20171112_IMG_0518. It has ta...\n",
       "4227    . It has title of 20171112_IMG_0531. It has ta...\n",
       "4228    . It has title of 20171112_IMG_0539. It has ta...\n",
       "4229                                   . It has title ...\n",
       "Name: text_data, Length: 2692, dtype: object"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_new['text_data']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e70bad25",
   "metadata": {},
   "source": [
    "Only 1963 out of the 2692 text data are unique to each other, i.e., not repetitive."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "fe3ce280",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1963"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_new['text_data'].unique())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ce7fc7d",
   "metadata": {},
   "source": [
    "#### Filtering out a sub-dataset with unique text data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bbe04b7d",
   "metadata": {},
   "source": [
    "Only the *FIRST* data point is kept while there are duplicates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "4cdeb550",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1963"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "candidates = df_new.drop_duplicates(subset='text_data',keep='first')\n",
    "len(candidates)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8346adc",
   "metadata": {},
   "source": [
    "### Detecting Different Languages in the Texts and Translating all to English"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "03528e37",
   "metadata": {},
   "source": [
    "#### Library Imports with Google Translator API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "7db718ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langdetect import detect\n",
    "from deep_translator import GoogleTranslator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "d2ed0574",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"i'm drinking coffee\""
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "translated = GoogleTranslator(source='auto', target='en').translate(\"我在喝咖啡\")\n",
    "translated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "d2dc358f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'zh-cn'"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "detect('你好')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "08508ea8",
   "metadata": {},
   "source": [
    "#### Text normalization and processing, Detecting Languages, and Performing Translations"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da38da7e",
   "metadata": {},
   "source": [
    "A new sentence occur when there is a period(.), exclamation mark(!), question mark(?) or semi-colon(;). The html signs, http addresses, formating elements of the texts, special quotation marks, and copyright symbol is removed. Only a sentence piece with no less than 3 words are considered for translation task.\n",
    "\n",
    "Italian is distinguished from the other non-English texts."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "cc17e424",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n"
     ]
    }
   ],
   "source": [
    "candi_dict_en = {}\n",
    "candi_dict_cn = {}\n",
    "candi_dict_cn_trans = {}\n",
    "candi_dict_other = {}\n",
    "candi_dict_other_trans = {}\n",
    "origin_en = []\n",
    "origin_cn = []\n",
    "origin_other = []\n",
    "origin_other_lan = []\n",
    "for i in candidates['text_data'].index:\n",
    "    for sent in candidates['text_data'][i].replace('St.', 'St').replace('S.', 'S').replace('!', '.').replace('?', '.').replace(';', '.').replace('。', '.').replace('！', '.').replace('？', '.').replace('；', '.').split('. '):\n",
    "        try:\n",
    "            sent = re.sub(r'\\<.*\\>', '', sent)\n",
    "            sent = re.sub(r'http[s]?://\\S+', '', sent)\n",
    "            sent = re.sub(r'--+', '', sent)\n",
    "            sent = re.sub(r'\\*+', '', sent)\n",
    "            sent = sent.replace('\\n', ' ').replace('&quot', '').replace('&amp', '').replace('#', '').replace('\\\\','')\n",
    "            sent = sent.replace('“', ' ').replace('„', ' ').replace('”', ' ')\n",
    "            sent = sent.replace('©', '')\n",
    "            for senty in sent.split('. '):\n",
    "                tt = 0\n",
    "                if len(senty.split())<3:\n",
    "                    continue\n",
    "                if senty.startswith('It has title of') or senty.startswith('It has tags of'):\n",
    "                    senti = ' '.join(senty.split()[4:])\n",
    "                    tt = 1\n",
    "                else:\n",
    "                    senti = senty\n",
    "                if detect(senti)=='en' and senty.strip() not in candi_dict_en:\n",
    "                    candi_dict_en[len(candi_dict_en)] = senty.strip()\n",
    "                    origin_en.append(i)\n",
    "                elif detect(senti) in ['zh-cn', 'zh-tw', 'ko', 'ja'] and senty.strip() not in candi_dict_cn:\n",
    "                    candi_dict_cn[len(candi_dict_cn)] = senty.strip()\n",
    "                    if tt==1:\n",
    "                        candi_dict_cn_trans[len(candi_dict_cn_trans)] = ' '.join(senty.split()[:4]) + ' ' + GoogleTranslator(source='auto', target='en').translate(senti.strip())\n",
    "                    else:\n",
    "                        candi_dict_cn_trans[len(candi_dict_cn_trans)] = GoogleTranslator(source='auto', target='en').translate(senti.strip())\n",
    "                    origin_cn.append(i)\n",
    "                elif detect(senti)!='it' and detect(senti)!='en' and senty.strip() not in candi_dict_other:\n",
    "                    candi_dict_other[len(candi_dict_other)] = senty.strip()\n",
    "                    if tt==1:\n",
    "                        candi_dict_other_trans[len(candi_dict_other_trans)] = ' '.join(senty.split()[:4]) + ' ' + GoogleTranslator(source='auto', target='en').translate(senti.strip())\n",
    "                    else:\n",
    "                        candi_dict_other_trans[len(candi_dict_other_trans)] = GoogleTranslator(source='auto', target='en').translate(senti.strip())\n",
    "                    origin_other.append(i)\n",
    "                    origin_other_lan.append(detect(senty))\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "            continue"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f21aeb6",
   "metadata": {},
   "source": [
    "In total, 497, 32, and 1609 sentences are detected as English, Chinese, and Other languages, respectively. After filtering out text data with less than 3 words, 1884 out of 1963 sentences remained."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "a15ae846",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(497, 32, 1609, 1884)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(origin_en)), len(set(origin_cn)), len(set(origin_other)), len(set(origin_en).union(set(origin_cn)).union(set(origin_other)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3aae7bf4",
   "metadata": {},
   "source": [
    "#### Filtering out Non-relevant Text data for Heritage Values and Attributes"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ed98be99",
   "metadata": {},
   "source": [
    "The stop word list includes camera brands, file names, advertisements, and place-holder formats for specific types of data (e.g., scanned from analog)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "d1c4a8a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "stoplist = ['sony', 'canon ', 'iphone','dilution','chesterton','leica','nikon','fuji','img','buy','steam','rolleiflex','ilford',\n",
    "            'sigma','e6', 'click', 'gear','luminance','fs','mercedes','spp','actv','atvo','samyang','thank','notes','twitter',\n",
    "            'gaz','oebb','follow','scan','voightlander','dsc', 'm1x', 'p10', 'f21', 'dhg', 'd01','l10', '_mg', '[','psx', 'zmr',\n",
    "            'discount', 'c_','shot','imag1','imag2','相片','photo', 'by:', 'how_', 'dpp', 'olympus']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "f5532db1",
   "metadata": {},
   "outputs": [],
   "source": [
    "filter_dict_en = {}\n",
    "filter_dict_cn = {}\n",
    "filter_dict_other = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "f57a90e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "origindicts = [candi_dict_en, candi_dict_cn_trans, candi_dict_other_trans]\n",
    "targdicts = [filter_dict_en, filter_dict_cn, filter_dict_other]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "7ba2f08e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2 It has title of 43B06C03-0A8C-4670-AA3D-1BF034F6B322_1_105_c sony\n",
      "3 It has title of 936635AE-45E9-4078-9DC8-4854241288C2_1_105_c sony\n",
      "4 It has title of 4894AD62-54A5-42C3-8BD6-9EDB9C1D0414_1_105_c sony\n",
      "15 Photo by Sony A7mII + Sony 24-105 F4 photo\n",
      "20 Photo by Sony A7mII + Sony 24-105 F4 photo\n",
      "26 Photo by Sony A7mII + Sony 24-105 F4 photo\n",
      "30 Photo by Sony A7mII + Sony 24-105 F4 photo\n",
      "182 It has title of 苏州IFS It has tags of sony\n",
      "187 It has title of 20190612_F0001: Sightseeing boat from Suzhou harbour sony\n",
      "191 It has title of 20190611_F0001: Busy touristy Shantang street sony\n",
      "195 It has title of 20190610_F0001: The phone squat, a blend of the traditional with the new sony\n",
      "200 It has title of 20190609_F0001: The fruit stand sony\n",
      "204 It has title of 20190608_F0001: Man on his electric scooter sony\n",
      "206 It has title of IMG_5539~photo sony\n",
      "207 It has title of IMG_5573~photo sony\n",
      "208 It has title of IMG_5579~photo sony\n",
      "209 It has title of IMG_5580~photo sony\n",
      "210 It has title of IMG_5584~photo sony\n",
      "211 It has title of IMG_5585~photo sony\n",
      "212 It has title of IMG_5586~photo sony\n",
      "213 It has title of IMG_5587~photo sony\n",
      "214 It has title of IMG_5588~photo sony\n",
      "215 It has title of IMG_5589~photo sony\n",
      "216 It has title of IMG_5593~photo sony\n",
      "217 It has title of IMG_5594~photo sony\n",
      "218 It has title of IMG_5595~photo sony\n",
      "219 It has title of IMG_5596~photo sony\n",
      "220 It has title of IMG_5598~photo sony\n",
      "221 It has title of IMG_5600~photo sony\n",
      "222 It has title of IMG_5606~photo sony\n",
      "223 It has title of IMG_5607~photo sony\n",
      "224 It has title of IMG_5608~photo sony\n",
      "225 It has title of IMG_5609~photo sony\n",
      "226 It has title of IMG_5610~photo sony\n",
      "227 It has title of IMG_5611~photo sony\n",
      "228 It has title of IMG_5617~photo sony\n",
      "229 It has title of IMG_5618~photo sony\n",
      "230 It has title of IMG_5620~photo sony\n",
      "231 It has title of IMG_5621~photo sony\n",
      "232 It has title of IMG_5622~photo sony\n",
      "233 It has title of IMG_5623~photo sony\n",
      "234 It has title of IMG_5624~photo sony\n",
      "235 It has title of IMG_5625~photo sony\n",
      "236 It has title of IMG_5626~photo sony\n",
      "237 It has title of IMG_5627~photo sony\n",
      "238 It has title of IMG_5628~photo sony\n",
      "239 It has title of IMG_5629~photo sony\n",
      "240 It has title of IMG_5630~photo sony\n",
      "241 It has title of IMG_5632~photo sony\n",
      "242 It has title of IMG_5635~photo sony\n",
      "243 It has title of IMG_5636~photo sony\n",
      "244 It has title of IMG_5637~photo sony\n",
      "245 It has title of IMG_5638~photo sony\n",
      "246 It has title of IMG_5639~photo sony\n",
      "247 It has title of IMG_5640~photo sony\n",
      "248 It has title of IMG_5641~photo sony\n",
      "249 It has title of IMG_5642~photo sony\n",
      "250 It has title of IMG_5643~photo sony\n",
      "251 It has title of IMG_5644~photo sony\n",
      "252 It has title of IMG_5645~photo sony\n",
      "253 It has title of IMG_5646~photo sony\n",
      "254 It has title of IMG_5647~photo sony\n",
      "255 It has title of IMG_5648~photo sony\n",
      "256 It has title of IMG_5649~photo sony\n",
      "257 It has title of IMG_5650~photo sony\n",
      "258 It has title of IMG_5651~photo sony\n",
      "259 It has title of IMG_5591~photo-full sony\n",
      "260 It has title of IMG_5635~photo-full sony\n",
      "266 It has title of IMG_20190319_161540_BURST001_COVER sony\n",
      "272 It has title of IMG_E4095 sony\n",
      "281 It has title of IMG_3414~photo sony\n",
      "282 It has title of IMG_3415~photo sony\n",
      "283 It has title of IMG_3416~photo sony\n",
      "284 It has title of IMG_3417~photo sony\n",
      "285 It has title of IMG_3418~photo sony\n",
      "286 It has title of IMG_3420~photo sony\n",
      "287 It has title of IMG_3421~photo sony\n",
      "288 It has title of IMG_3422~photo sony\n",
      "289 It has title of IMG_3423~photo sony\n",
      "290 It has title of IMG_3424~photo sony\n",
      "291 It has title of IMG_3427~photo sony\n",
      "292 It has title of IMG_3428~photo sony\n",
      "293 It has title of IMG_3430~photo sony\n",
      "294 It has title of IMG_3433~photo sony\n",
      "295 It has title of IMG_3439~photo sony\n",
      "296 It has title of IMG_3440~photo sony\n",
      "297 It has title of IMG_3441~photo sony\n",
      "298 It has title of IMG_3442~photo sony\n",
      "299 It has title of IMG_3444~photo sony\n",
      "300 It has title of IMG_3445~photo sony\n",
      "301 It has title of IMG_3446~photo sony\n",
      "302 It has title of IMG_3447~photo sony\n",
      "303 It has title of IMG_3448~photo sony\n",
      "304 It has title of IMG_3449~photo sony\n",
      "305 It has title of IMG_3451~photo sony\n",
      "306 It has title of IMG_3452~photo sony\n",
      "307 It has title of IMG_3453~photo sony\n",
      "308 It has title of IMG_3455~photo sony\n",
      "309 It has title of IMG_3461~photo sony\n",
      "310 It has title of IMG_3470~photo sony\n",
      "311 It has title of IMG_3472~photo sony\n",
      "312 It has title of IMG_3411~photo sony\n",
      "313 It has title of IMG_3412~photo sony\n",
      "314 It has title of IMG_3443~photo sony\n",
      "315 It has title of IMG_3413~photo sony\n",
      "370 BY: Brian Lou PROGRAM: N/A LOCATION: Suzhou, China sony\n",
      "374 It has title of 20180725_Suzhou_Garden of the Humble Administrator_38n sony\n",
      "376 It has title of 20180725_Suzhou_Garden of the Humble Administrator_1 sony\n",
      "378 It has title of 20180725_Suzhou_Garden of the Humble Administrator_1w sony\n",
      "380 It has title of 20180725_Suzhou_Garden of the Humble Administrator_2 sony\n",
      "382 It has title of 20180725_Suzhou_Garden of the Humble Administrator_2n sony\n",
      "384 It has title of 20180725_Suzhou_Garden of the Humble Administrator_2q sony\n",
      "386 It has title of 20180725_Suzhou_Garden of the Humble Administrator_3 sony\n",
      "388 It has title of 20180725_Suzhou_Garden of the Humble Administrator_4 sony\n",
      "390 It has title of 20180725_Suzhou_Garden of the Humble Administrator_5 sony\n",
      "392 It has title of 20180725_Suzhou_Garden of the Humble Administrator_6 sony\n",
      "394 It has title of 20180725_Suzhou_Garden of the Humble Administrator_7 sony\n",
      "396 It has title of 20180725_Suzhou_Garden of the Humble Administrator_8 sony\n",
      "398 It has title of 20180725_Suzhou_Garden of the Humble Administrator_9 sony\n",
      "400 It has title of 20180725_Suzhou_Garden of the Humble Administrator_10n sony\n",
      "402 It has title of 20180725_Suzhou_Garden of the Humble Administrator_10w sony\n",
      "404 It has title of 20180725_Suzhou_Garden of the Humble Administrator_11 sony\n",
      "406 It has title of 20180725_Suzhou_Garden of the Humble Administrator_11w sony\n",
      "408 It has title of 20180725_Suzhou_Garden of the Humble Administrator_12 sony\n",
      "410 It has title of 20180725_Suzhou_Garden of the Humble Administrator_13 sony\n",
      "412 It has title of 20180725_Suzhou_Garden of the Humble Administrator_14 sony\n",
      "414 It has title of 20180725_Suzhou_Garden of the Humble Administrator_15 sony\n",
      "416 It has title of 20180725_Suzhou_Garden of the Humble Administrator_16 sony\n",
      "418 It has title of 20180725_Suzhou_Garden of the Humble Administrator_17 sony\n",
      "420 It has title of 20180725_Suzhou_Garden of the Humble Administrator_17w sony\n",
      "422 It has title of 20180725_Suzhou_Garden of the Humble Administrator_18 sony\n",
      "424 It has title of 20180725_Suzhou_Garden of the Humble Administrator_19 sony\n",
      "426 It has title of 20180725_Suzhou_Garden of the Humble Administrator_20 sony\n",
      "428 It has title of 20180725_Suzhou_Garden of the Humble Administrator_21 sony\n",
      "430 It has title of 20180725_Suzhou_Garden of the Humble Administrator_22 sony\n",
      "432 It has title of 20180725_Suzhou_Garden of the Humble Administrator_22h sony\n",
      "434 It has title of 20180725_Suzhou_Garden of the Humble Administrator_22n sony\n",
      "436 It has title of 20180725_Suzhou_Garden of the Humble Administrator_23 sony\n",
      "438 It has title of 20180725_Suzhou_Garden of the Humble Administrator_23n sony\n",
      "440 It has title of 20180725_Suzhou_Garden of the Humble Administrator_24 sony\n",
      "444 It has title of 2018 China snap shots sony\n",
      "445 It has title of 2018 china family-72 sony\n",
      "536 It has title of 20190606_F0001: Electric scooters taking over the bike lanes sony\n",
      "538 It has title of IMG_5540~photo sony\n",
      "539 It has title of IMG_5542~photo sony\n",
      "540 It has title of IMG_5543~photo sony\n",
      "541 It has title of IMG_5544~photo sony\n",
      "542 It has title of IMG_5545~photo sony\n",
      "543 It has title of IMG_5549~photo sony\n",
      "544 It has title of IMG_5550~photo sony\n",
      "545 It has title of IMG_5551~photo sony\n",
      "546 It has title of IMG_5552~photo sony\n",
      "547 It has title of IMG_5555~photo sony\n",
      "548 It has title of IMG_5557~photo sony\n",
      "549 It has title of IMG_5561~photo sony\n",
      "550 It has title of IMG_5562~photo sony\n",
      "551 It has title of IMG_5563~photo sony\n",
      "552 It has title of IMG_5564~photo sony\n",
      "553 It has title of IMG_5566~photo sony\n",
      "554 It has title of IMG_5567~photo sony\n",
      "555 It has title of IMG_5568~photo sony\n",
      "556 It has title of IMG_5570~photo sony\n",
      "557 It has title of IMG_5571~photo sony\n",
      "558 It has title of IMG_5553~photo sony\n",
      "559 It has title of IMG_5565~photo sony\n",
      "562 It has title of 20190523_F0001: Chinese wedding photography sony\n",
      "566 It has title of 20190521_F0001: Seven-mile Shantang sony\n",
      "570 It has title of 20190520_F0001: Sketching the tree sony\n",
      "574 It has title of 20190519_F0001: It's all about the rocks sony\n",
      "577 It has title of 20190518_F0001: A mini garden within the Lingering Garden sony\n",
      "582 It has title of IMG_E2656 sony\n",
      "583 It has title of IMG_E2740 sony\n",
      "584 It has title of IMG_3459~photo sony\n",
      "585 It has title of IMG_3407~photo sony\n",
      "586 It has title of IMG_3408~photo sony\n",
      "590 It has title of 20180725_Suzhou_Garden of the Humble Administrator_31 sony\n",
      "592 It has title of 20180725_Suzhou_Garden of the Humble Administrator_31q sony\n",
      "594 It has title of 20180725_Suzhou_Garden of the Humble Administrator_32 sony\n",
      "596 It has title of 20180725_Suzhou_Garden of the Humble Administrator_32h sony\n",
      "598 It has title of 20180725_Suzhou_Garden of the Humble Administrator_32q sony\n",
      "600 It has title of 20180725_Suzhou_Garden of the Humble Administrator_34 sony\n",
      "602 It has title of 20180725_Suzhou_Garden of the Humble Administrator_35 sony\n",
      "604 It has title of 20180725_Suzhou_Garden of the Humble Administrator_36 sony\n",
      "606 It has title of 20180725_Suzhou_Garden of the Humble Administrator_37h sony\n",
      "608 It has title of 20180725_Suzhou_Garden of the Humble Administrator_37n sony\n",
      "610 It has title of 20180725_Suzhou_Garden of the Humble Administrator_37w sony\n",
      "612 It has title of 20180725_Suzhou_Garden of the Humble Administrator_38 sony\n",
      "617 It has title of 20180725_Suzhou_Garden of the Humble Administrator_29 sony\n",
      "619 It has title of 20180725_Suzhou_Garden of the Humble Administrator_30 sony\n",
      "629 It has title of IMG_E0288 sony\n",
      "655 It has title of IMG_20190319_144335_BURST001_COVER sony\n",
      "656 It has title of IMG_20190319_161314_BURST001_COVER sony\n",
      "657 BY: Brian Lou PROGRAM: N/A LOCATION: Suzhou, China sony\n",
      "659 It has title of 20180725_Suzhou_Garden of the Humble Administrator_24n sony\n",
      "661 It has title of 20180725_Suzhou_Garden of the Humble Administrator_26 sony\n",
      "663 It has title of 20180725_Suzhou_Garden of the Humble Administrator_27 sony\n",
      "665 It has title of 20180725_Suzhou_Garden of the Humble Administrator_28 sony\n",
      "672 It has title of iPhone iphone\n",
      "697 It has title of IMG_E2718 sony\n",
      "698 It has title of 20180725_Suzhou_Garden of the Humble Administrator_8w sony\n",
      "700 It has title of 20180725_Suzhou_Garden of the Humble Administrator_25 sony\n",
      "26 It has title of Photos 2017-4-2 19 54 55 photo\n",
      "27 It has title of Photos 2017-4-2 11 28 38 photo\n",
      "28 It has title of Photos 2017-4-2 17 04 20 photo\n",
      "29 It has title of Photos 2017-4-2 14 40 10 photo\n",
      "30 It has title of Photos 2017-4-2 17 59 14 photo\n",
      "2 It has title of 3F886916-F7F9-4723-82DE-20F3C4DF2C40_1_105_c sony\n",
      "3 It has title of 057F3E9D-C4AA-4454-A214-31EF82EB31AB_1_105_c sony\n",
      "25 It has title of IMG_20200510_152121 sony\n",
      "26 It has title of IMG_20200510_152137 sony\n",
      "27 It has title of IMG_20200510_152331 sony\n",
      "28 It has title of IMG_20200510_152354 sony\n",
      "29 It has title of IMG_20200510_152405 sony\n",
      "30 It has title of IMG_20200510_153702 sony\n",
      "31 It has title of IMG_20200510_151736 sony\n",
      "32 It has title of IMG_20200510_151747 sony\n",
      "33 It has title of IMG_20200510_152419 sony\n",
      "34 It has title of IMG_20200510_152511 sony\n",
      "35 It has title of IMG_20200510_152740 sony\n",
      "36 It has title of IMG_20200510_153546 sony\n",
      "37 It has title of IMG_20200510_153717 sony\n",
      "38 It has title of IMG_20200510_150351 sony\n",
      "39 It has title of IMG_20200510_150610 sony\n",
      "40 It has title of IMG_20200510_150204 sony\n",
      "41 It has title of IMG_20200510_150242 sony\n",
      "42 It has title of IMG_20200510_150317 sony\n",
      "43 It has title of 00001IMG_00001_BURST20200510150230 sony\n",
      "44 It has title of IMG_20200510_114206 sony\n",
      "46 It has title of HOW_6155 sony\n",
      "47 It has title of HOW_6173 sony\n",
      "48 It has title of HOW_5813 sony\n",
      "49 It has title of HOW_5814 sony\n",
      "50 It has title of HOW_5820 sony\n",
      "51 It has title of HOW_5822 sony\n",
      "52 It has title of HOW_5824 sony\n",
      "53 It has title of HOW_5832 sony\n",
      "54 It has title of HOW_5838 sony\n",
      "55 It has title of HOW_5842 sony\n",
      "56 It has title of HOW_5845 sony\n",
      "57 It has title of HOW_5846 sony\n",
      "59 It has title of HOW_5857 sony\n",
      "60 It has title of HOW_5858 sony\n",
      "61 It has title of HOW_5859 sony\n",
      "62 It has title of HOW_5860 sony\n",
      "63 It has title of HOW_5862 sony\n",
      "64 It has title of HOW_5873 sony\n",
      "65 It has title of HOW_5874 sony\n",
      "66 It has title of HOW_5887 sony\n",
      "67 It has title of HOW_5898 sony\n",
      "68 It has title of HOW_5901 sony\n",
      "69 It has title of HOW_5903 sony\n",
      "70 It has title of HOW_5904 sony\n",
      "71 It has title of HOW_5910 sony\n",
      "72 It has title of HOW_5911 sony\n",
      "73 It has title of HOW_5914 sony\n",
      "74 It has title of HOW_5919 sony\n",
      "75 It has title of HOW_5924 sony\n",
      "76 It has title of HOW_6012 sony\n",
      "77 It has title of HOW_6021 sony\n",
      "78 It has title of HOW_5736 sony\n",
      "79 It has title of HOW_5743 sony\n",
      "80 It has title of HOW_5746 sony\n",
      "81 It has title of HOW_5751 sony\n",
      "82 It has title of HOW_5756 sony\n",
      "83 It has title of HOW_5775 sony\n",
      "84 It has title of HOW_5778 sony\n",
      "85 It has title of HOW_5779 sony\n",
      "86 It has title of HOW_5796 sony\n",
      "87 It has title of HOW_5800 sony\n",
      "88 It has title of HOW_5801 sony\n",
      "89 It has title of HOW_5803 sony\n",
      "90 It has title of HOW_5806 sony\n",
      "91 It has title of HOW_5807 sony\n",
      "92 It has title of HOW_5808 sony\n",
      "93 It has title of HOW_5809 sony\n",
      "94 It has title of HOW_5810 sony\n",
      "95 It has title of HOW_5811 sony\n",
      "96 It has title of HOW_5908 sony\n",
      "97 It has title of HOW_6081 sony\n",
      "98 It has title of HOW_6085 sony\n",
      "99 It has title of HOW_5812 sony\n",
      "100 It has title of HOW_6087 sony\n",
      "103 It has title of 66791-Suzhou sony\n",
      "105 It has title of IMG_4104 sony\n",
      "106 It has title of IMG_4105 sony\n",
      "107 It has title of IMG_20200104_144948 sony\n",
      "108 It has title of IMG_20200104_145141 sony\n",
      "109 It has title of IMG_20200104_145426 sony\n",
      "110 It has title of IMG_20200104_145433 sony\n",
      "111 It has title of IMG_20200104_150155 sony\n",
      "112 It has title of IMG_20200104_151728 sony\n",
      "113 It has title of IMG_20200106_000723 sony\n",
      "114 It has title of 66641-Suzhou sony\n",
      "115 It has title of 66632-Suzhou sony\n",
      "116 It has title of 66626-Suzhou sony\n",
      "117 It has title of 66617-Suzhou sony\n",
      "118 It has title of 66597-Suzhou sony\n",
      "119 It has title of 66232-Suzhou sony\n",
      "120 It has title of 66399-Suzhou sony\n",
      "122 It has title of 66700-Suzhou sony\n",
      "123 It has title of 66570-Suzhou sony\n",
      "124 It has title of 66334-Suzhou sony\n",
      "125 It has title of 66559-Suzhou sony\n",
      "126 It has title of 66234-Suzhou sony\n",
      "127 It has title of 48290-Suzhou sony\n",
      "128 It has title of 48232-Suzhou sony\n",
      "129 It has title of 48336-Suzhou sony\n",
      "130 It has title of 66696-Suzhou sony\n",
      "132 It has title of 66692-Suzhou sony\n",
      "134 It has title of 66691-Suzhou sony\n",
      "136 It has title of 66684-Suzhou sony\n",
      "138 It has title of 66686-Suzhou sony\n",
      "140 It has title of 66410-Suzhou sony\n",
      "142 It has title of 66401-Suzhou sony\n",
      "144 It has title of 66397-Suzhou sony\n",
      "146 It has title of 66394-Suzhou sony\n",
      "148 It has title of 66382-Suzhou sony\n",
      "150 It has title of 66388-Suzhou sony\n",
      "152 It has title of 66390-Suzhou sony\n",
      "154 It has title of 66363-Suzhou sony\n",
      "156 It has title of 66377-Suzhou sony\n",
      "158 It has title of 66374-Suzhou sony\n",
      "160 It has title of 66367-Suzhou sony\n",
      "162 It has title of 66793-Suzhou sony\n",
      "163 It has title of 66792-Suzhou sony\n",
      "164 It has title of 66789-Suzhou sony\n",
      "165 It has title of 66785-Suzhou sony\n",
      "166 It has title of 66776-Suzhou sony\n",
      "167 It has title of 66779-Suzhou sony\n",
      "168 It has title of 66769-Suzhou sony\n",
      "169 It has title of 66728-Suzhou sony\n",
      "170 It has title of 66735-Suzhou sony\n",
      "171 It has title of 66725-Suzhou sony\n",
      "172 It has title of 66721-Suzhou sony\n",
      "173 It has title of 66710-Suzhou sony\n",
      "174 It has title of 66677-Suzhou sony\n",
      "175 It has title of 66672-Suzhou sony\n",
      "176 It has title of 66665-Suzhou sony\n",
      "178 It has title of 66658-Suzhou sony\n",
      "179 It has title of 66655-Suzhou sony\n",
      "180 It has title of 66653-Suzhou sony\n",
      "181 It has title of 66652-Suzhou sony\n",
      "182 It has title of 66514-Suzhou sony\n",
      "183 It has title of 66502-Suzhou sony\n",
      "185 It has title of 66510-Suzhou sony\n",
      "186 It has title of 66501-Suzhou sony\n",
      "187 It has title of 66486-Suzhou sony\n",
      "188 It has title of 66739-Suzhou sony\n",
      "189 It has title of 66670-Suzhou sony\n",
      "190 It has title of 66498-Suzhou sony\n",
      "191 It has title of 66495-Suzhou sony\n",
      "192 It has title of 66489-Suzhou sony\n",
      "193 It has title of 66442-Suzhou sony\n",
      "195 It has title of 66444-Suzhou sony\n",
      "197 It has title of 66438-Suzhou sony\n",
      "199 It has title of 66440-Suzhou sony\n",
      "200 It has title of 66434-Suzhou sony\n",
      "202 It has title of 66746-Suzhou sony\n",
      "203 It has title of 66738-Suzhou sony\n",
      "204 It has title of 66743-Suzhou sony\n",
      "205 It has title of 66741-Suzhou sony\n",
      "206 It has title of 66736-Suzhou sony\n",
      "207 It has title of 48396-Suzhou sony\n",
      "208 It has title of 66300-Suzhou sony\n",
      "209 It has title of 48452-Suzhou sony\n",
      "210 It has title of 48510-Suzhou sony\n",
      "211 It has title of 48292-Suzhou sony\n",
      "212 It has title of 48228-Suzhou sony\n",
      "213 It has title of 48248-Suzhou sony\n",
      "214 It has title of 48238-Suzhou sony\n",
      "215 It has title of 48186-Suzhou sony\n",
      "216 It has title of 48178-Suzhou sony\n",
      "217 It has title of 48182-Suzhou sony\n",
      "218 It has title of 48184-Suzhou sony\n",
      "219 It has title of 66335-Suzhou sony\n",
      "220 It has title of 66577-Suzhou sony\n",
      "221 It has title of 66332-Suzhou sony\n",
      "222 It has title of 66317-Suzhou sony\n",
      "223 It has title of 66331-Suzhou sony\n",
      "224 It has title of 66319-Suzhou sony\n",
      "225 It has title of 66326-Suzhou sony\n",
      "226 It has title of 66309-Suzhou sony\n",
      "227 It has title of 66308-Suzhou sony\n",
      "228 It has title of 66313-Suzhou sony\n",
      "229 It has title of 66304-Suzhou sony\n",
      "230 It has title of 66697-Suzhou sony\n",
      "232 It has title of 66694-Suzhou sony\n",
      "234 It has title of 66690-Suzhou sony\n",
      "236 It has title of 66403-Suzhou sony\n",
      "238 It has title of 66681-Suzhou sony\n",
      "240 It has title of 66398-Suzhou sony\n",
      "242 It has title of 66400-Suzhou sony\n",
      "244 It has title of 66396-Suzhou sony\n",
      "246 It has title of 66393-Suzhou sony\n",
      "248 It has title of 66389-Suzhou sony\n",
      "250 It has title of 66386-Suzhou sony\n",
      "252 It has title of 66380-Suzhou sony\n",
      "254 It has title of 66364-Suzhou sony\n",
      "256 It has title of 66376-Suzhou sony\n",
      "258 It has title of 66372-Suzhou sony\n",
      "260 It has title of 66716-Suzhou sony\n",
      "261 It has title of 48480-Suzhou sony\n",
      "262 It has title of 48354-Suzhou sony\n",
      "263 It has title of 48340-Suzhou sony\n",
      "264 It has title of 48332-Suzhou sony\n",
      "265 It has title of 48330-Suzhou sony\n",
      "266 It has title of 48296-Suzhou sony\n",
      "267 It has title of 66688-Suzhou sony\n",
      "269 It has title of 48352-Suzhou sony\n",
      "270 It has title of 48302-Suzhou sony\n",
      "271 It has title of 48316-Suzhou sony\n",
      "272 It has title of 48266-Suzhou sony\n",
      "273 It has title of 48252-Suzhou sony\n",
      "274 It has title of 48244-Suzhou sony\n",
      "275 It has title of 48246-Suzhou sony\n",
      "276 It has title of 66693-Suzhou sony\n",
      "278 It has title of 66649-Suzhou sony\n",
      "279 It has title of 66634-Suzhou sony\n",
      "280 It has title of 66561-Suzhou sony\n",
      "281 It has title of 66580-Suzhou sony\n",
      "282 It has title of 66536-Suzhou sony\n",
      "283 It has title of 66523-Suzhou sony\n",
      "284 It has title of 66506-Suzhou sony\n",
      "285 It has title of 66435-Suzhou sony\n",
      "287 It has title of 66417-Suzhou sony\n",
      "288 It has title of 66299-Suzhou sony\n",
      "289 It has title of 66383-Suzhou sony\n",
      "291 It has title of 48230-Suzhou sony\n",
      "292 It has title of 48628-Suzhou sony\n",
      "293 It has title of 66717-Suzhou sony\n",
      "294 It has title of 48516-Suzhou sony\n",
      "295 It has title of 48512-Suzhou sony\n",
      "296 It has title of 66482-Suzhou sony\n",
      "297 It has title of 48190-Suzhou sony\n",
      "298 It has title of 48504-Suzhou sony\n",
      "299 It has title of 48500-Suzhou sony\n",
      "300 It has title of 48494-Suzhou sony\n",
      "301 It has title of 48470-Suzhou sony\n",
      "302 It has title of 48482-Suzhou sony\n",
      "303 It has title of 48466-Suzhou sony\n",
      "304 It has title of 48294-Suzhou sony\n",
      "305 It has title of 48258-Suzhou sony\n",
      "306 It has title of 48306-Suzhou sony\n",
      "307 It has title of 48260-Suzhou sony\n",
      "308 It has title of 66483-Suzhou sony\n",
      "309 It has title of 66480-Suzhou sony\n",
      "310 It has title of 66476-Suzhou sony\n",
      "311 It has title of 66466-Suzhou sony\n",
      "312 It has title of 66467-Suzhou sony\n",
      "313 It has title of 66470-Suzhou sony\n",
      "314 It has title of 66465-Suzhou sony\n",
      "315 It has title of 66454-Suzhou sony\n",
      "316 It has title of 66456-Suzhou sony\n",
      "317 It has title of 66460-Suzhou sony\n",
      "318 It has title of 66451-Suzhou sony\n",
      "326 It has title of IMG_4206 sony\n",
      "327 It has title of IMG_4207 sony\n",
      "328 It has title of IMG_4208 sony\n",
      "329 It has title of IMG_4209 sony\n",
      "330 It has title of IMG_1281 sony\n",
      "331 It has title of IMG_1279 sony\n",
      "332 It has title of IMG_1280 sony\n",
      "333 It has title of IMG_1241 sony\n",
      "334 It has title of IMG_1255 sony\n",
      "335 It has title of IMG_1257 sony\n",
      "336 It has title of IMG_1242 sony\n",
      "337 It has title of IMG_1243 sony\n",
      "338 It has title of IMG_1235 sony\n",
      "339 It has title of IMG_1236 sony\n",
      "340 It has title of IMG_1238 sony\n",
      "341 It has title of IMG_1239 sony\n",
      "344 It has title of IMG_3239 sony\n",
      "345 It has title of IMG_3240 sony\n",
      "346 It has title of IMG_3241 sony\n",
      "347 It has title of IMG_3244 sony\n",
      "348 It has title of IMG_3245 sony\n",
      "349 It has title of IMG_3247 sony\n",
      "350 It has title of IMG_3248 sony\n",
      "351 It has title of IMG_3249 sony\n",
      "352 It has title of IMG_3250 sony\n",
      "353 It has title of IMG_3251 sony\n",
      "354 It has title of IMG_4488 sony\n",
      "355 It has title of IMG_4489 sony\n",
      "356 It has title of IMG_3232 sony\n",
      "357 It has title of IMG_3233 sony\n",
      "358 It has title of IMG_3051 sony\n",
      "359 It has title of 20190522_F0001: Seven-mile Shantang lanterns sony\n",
      "360 It has title of IMG_3005 sony\n",
      "361 It has title of IMG_3007 sony\n",
      "362 It has title of IMG_3008 sony\n",
      "363 It has title of IMG_2894 sony\n",
      "364 It has title of IMG_2895 sony\n",
      "365 It has title of IMG_2896 sony\n",
      "366 It has title of IMG_2897 sony\n",
      "367 It has title of IMG_2874 sony\n",
      "368 It has title of IMG_2873 sony\n",
      "369 It has title of IMG_2872 sony\n",
      "370 It has title of IMG_2871 sony\n",
      "371 It has title of IMG_2862 sony\n",
      "372 It has title of IMG_2863 sony\n",
      "373 It has title of IMG_2846 sony\n",
      "374 It has title of IMG_2847 sony\n",
      "375 It has title of IMG_2848 sony\n",
      "376 It has title of IMG_2849 sony\n",
      "377 It has title of IMG_2850 sony\n",
      "378 It has title of IMG_2851 sony\n",
      "379 It has title of IMG_2817 sony\n",
      "380 It has title of IMG_2816 sony\n",
      "381 It has title of IMG_2708 sony\n",
      "382 It has title of IMG_2709 sony\n",
      "383 It has title of IMG_2710 sony\n",
      "384 It has title of IMG_2711 sony\n",
      "385 It has title of IMG_2815 sony\n",
      "388 It has title of IMG_20190319_161314_BURST006 sony\n",
      "390 It has title of IMG_4542 sony\n",
      "392 It has title of IMG_7279 sony\n",
      "393 It has title of IMG_7281 sony\n",
      "394 It has title of IMG_7280 sony\n",
      "395 It has title of IMG_7259 sony\n",
      "396 It has title of IMG_7258 sony\n",
      "397 It has title of IMG_7252 sony\n",
      "398 It has title of IMG_7254 sony\n",
      "399 It has title of IMG_7255 sony\n",
      "400 It has title of IMG_7242 sony\n",
      "401 It has title of IMG_7243 sony\n",
      "402 It has title of IMG_7245 sony\n",
      "403 It has title of IMG_7246 sony\n",
      "404 It has title of IMG_7247 sony\n",
      "405 It has title of IMG_8495 sony\n",
      "406 It has title of IMG_8496 sony\n",
      "407 It has title of IMG_8497 sony\n",
      "408 It has title of IMG_8498 sony\n",
      "409 It has title of IMG_8499 sony\n",
      "410 It has title of IMG_8500 sony\n",
      "411 It has title of IMG_8501 sony\n",
      "412 It has title of IMG_8502 sony\n",
      "413 It has title of IMG_8487 sony\n",
      "414 It has title of IMG_8488 sony\n",
      "415 It has title of IMG_8483 sony\n",
      "416 It has title of IMG_8484 sony\n",
      "417 It has title of IMG_8485 sony\n",
      "418 It has title of IMG_8486 sony\n",
      "419 It has title of IMG_4094 sony\n",
      "420 It has title of IMG_4095 sony\n",
      "421 It has title of IMG_4090 sony\n",
      "422 It has title of IMG_4091 sony\n",
      "423 It has title of IMG_4093 sony\n",
      "424 It has title of IMG_4078 sony\n",
      "425 It has title of IMG_4079 sony\n",
      "426 It has title of IMG_4081 sony\n",
      "427 It has title of IMG_3881 sony\n",
      "428 It has title of IMG_3882 sony\n",
      "429 It has title of IMG_3883 sony\n",
      "430 It has title of IMG_3884 sony\n",
      "431 It has title of IMG_3872 sony\n",
      "432 It has title of IMG_3873 sony\n",
      "433 It has title of IMG_3874 sony\n",
      "434 It has title of IMG_3875 sony\n",
      "435 It has title of IMG_3876 sony\n",
      "436 It has title of IMG_3868 sony\n",
      "437 It has title of IMG_3869 sony\n",
      "438 It has title of IMG_7509 sony\n",
      "439 It has title of IMG_7510 sony\n",
      "440 It has title of IMG_7353 sony\n",
      "441 It has title of IMG_7354 sony\n",
      "442 It has title of IMG_7355 sony\n",
      "443 It has title of IMG_7356 sony\n",
      "444 It has title of IMG_7357 sony\n",
      "445 It has title of IMG_7358 sony\n",
      "446 It has title of IMG_7359 sony\n",
      "447 It has title of IMG_7360 sony\n",
      "448 It has title of IMG_7343 sony\n",
      "449 It has title of IMG_7344 sony\n",
      "450 It has title of IMG_7345 sony\n",
      "451 It has title of IMG_7347 sony\n",
      "452 It has title of IMG_7348 sony\n",
      "453 It has title of IMG_7315 sony\n",
      "454 It has title of 20190217-MVIMG_20190217_064851 sony\n",
      "455 It has title of IMG_7253 sony\n",
      "456 It has title of IMG_7248 sony\n",
      "457 It has title of IMG_7225 sony\n",
      "458 It has title of IMG_7226 sony\n",
      "459 It has title of IMG_3630 sony\n",
      "460 It has title of IMG_3631 sony\n",
      "461 It has title of IMG_3625 sony\n",
      "462 It has title of IMG_3626 sony\n",
      "463 It has title of IMG_3612 sony\n",
      "464 It has title of IMG_3615 sony\n",
      "465 It has title of IMG_3616 sony\n",
      "466 It has title of IMG_3617 sony\n",
      "467 It has title of IMG_3618 sony\n",
      "468 It has title of IMG_3613 sony\n",
      "469 It has title of IMG_3614 sony\n",
      "470 It has title of IMG_3585 sony\n",
      "471 It has title of IMG_3584 sony\n",
      "472 It has title of IMG_3586 sony\n",
      "473 It has title of IMG_3587 sony\n",
      "474 It has title of IMG_3171 sony\n",
      "475 It has title of IMG_3172 sony\n",
      "476 It has title of IMG_3164 sony\n",
      "477 It has title of IMG_3150 sony\n",
      "478 It has title of IMG_3165 sony\n",
      "479 It has title of IMG_3151 sony\n",
      "480 It has title of IMG_3166 sony\n",
      "481 It has title of IMG_3152 sony\n",
      "482 It has title of IMG_3167 sony\n",
      "483 It has title of IMG_3153 sony\n",
      "484 It has title of IMG_3168 sony\n",
      "485 It has title of IMG_3169 sony\n",
      "486 It has title of IMG_3154 sony\n",
      "487 It has title of IMG_3155 sony\n",
      "488 It has title of IMG_3156 sony\n",
      "489 It has title of IMG_3157 sony\n",
      "490 It has title of IMG_3158 sony\n",
      "491 It has title of IMG_3159 sony\n",
      "492 It has title of IMG_3160 sony\n",
      "493 It has title of IMG_3161 sony\n",
      "494 It has title of IMG_3162 sony\n",
      "495 It has title of IMG_3163 sony\n",
      "496 It has title of IMG_3103 sony\n",
      "497 It has title of IMG_3104 sony\n",
      "498 It has title of IMG_3071 sony\n",
      "499 It has title of IMG_3067 sony\n",
      "500 It has title of IMG_3068 sony\n",
      "501 It has title of IMG_3069 sony\n",
      "502 It has title of IMG_3092 sony\n",
      "503 It has title of IMG_3055 sony\n",
      "504 It has title of IMG_3056 sony\n",
      "505 It has title of IMG_3057 sony\n",
      "506 It has title of IMG_3058 sony\n",
      "507 It has title of IMG_3059 sony\n",
      "508 It has title of IMG_3060 sony\n",
      "509 It has title of IMG_3061 sony\n",
      "510 It has title of IMG_3062 sony\n",
      "511 It has title of IMG_3063 sony\n",
      "512 It has title of IMG_3064 sony\n",
      "513 It has title of IMG_3065 sony\n",
      "514 It has title of IMG_3066 sony\n",
      "515 It has title of IMG_2972 sony\n",
      "516 It has title of IMG_2973 sony\n",
      "517 It has title of IMG_2765 sony\n",
      "518 It has title of IMG_6374 sony\n",
      "519 It has title of IMG_6369 sony\n",
      "520 It has title of IMG_6370 sony\n",
      "521 It has title of IMG_6371 sony\n",
      "522 It has title of IMG_6372 sony\n",
      "528 It has title of IMG_1104 sony\n",
      "530 It has title of IMG_1105 sony\n",
      "532 It has title of IMG_1106 sony\n",
      "534 It has title of IMG_1107 sony\n",
      "536 It has title of IMG_1109 sony\n",
      "538 It has title of IMG_1114 sony\n",
      "542 It has title of DSCF9936 sony\n",
      "543 It has title of DSCF9948 sony\n",
      "646 It has title of IMG_3009 sony\n",
      "647 It has title of IMG_3010 sony\n",
      "648 It has title of IMG_3011 sony\n",
      "649 It has title of IMG_3012 sony\n",
      "650 It has title of IMG_3013 sony\n",
      "651 It has title of IMG_3238 sony\n",
      "652 It has title of IMG_3014 sony\n",
      "653 It has title of IMG_4315 sony\n",
      "654 It has title of IMG_4316 sony\n",
      "655 It has title of IMG_3006 sony\n",
      "656 It has title of IMG_2283 sony\n",
      "657 It has title of IMG_2284 sony\n",
      "658 It has title of IMG_2285 sony\n",
      "659 It has title of IMG_2282 sony\n",
      "660 It has title of IMG_2232 sony\n",
      "661 It has title of IMG_2233 sony\n",
      "662 It has title of IMG_5076 sony\n",
      "663 It has title of IMG_5077 sony\n",
      "664 It has title of IMG_5078 sony\n",
      "665 It has title of IMG_5079 sony\n",
      "666 It has title of IMG_5080 sony\n",
      "667 It has title of IMG_5081 sony\n",
      "668 It has title of IMG_5082 sony\n",
      "669 It has title of IMG_5083 sony\n",
      "670 It has title of IMG_5084 sony\n",
      "671 It has title of IMG_5085 sony\n",
      "672 It has title of IMG_3070 sony\n",
      "673 It has title of IMG_3052 sony\n",
      "674 It has title of IMG_3053 sony\n",
      "675 It has title of IMG_3054 sony\n",
      "676 It has title of IMG_2859 sony\n",
      "677 It has title of IMG_2860 sony\n",
      "678 It has title of IMG_2779 sony\n",
      "679 It has title of IMG_2780 sony\n",
      "680 It has title of IMG_2852 sony\n",
      "681 It has title of IMG_2853 sony\n",
      "682 It has title of IMG_1831 sony\n",
      "683 It has title of IMG_1832 sony\n",
      "688 It has title of IMG_1639 sony\n",
      "689 It has title of IMG_1640 sony\n",
      "690 It has title of IMG_1320 sony\n",
      "691 It has title of IMG_1638 sony\n",
      "692 It has title of IMG_1321 sony\n",
      "693 It has title of IMG_1309 sony\n",
      "694 It has title of IMG_1310 sony\n",
      "695 It has title of IMG_0631 sony\n",
      "696 It has title of IMG_0638 sony\n",
      "697 It has title of IMG_0639 sony\n",
      "698 It has title of IMG_0640 sony\n",
      "699 It has title of IMG_0641 sony\n",
      "700 It has title of IMG_0649 sony\n",
      "701 It has title of IMG_0620 sony\n",
      "702 It has title of IMG_0621 sony\n",
      "703 It has title of DSC01139 sony\n",
      "705 It has title of DSC01140 sony\n",
      "707 It has title of DSC01145 sony\n",
      "709 It has title of DSC01145a sony\n",
      "711 It has title of DSC01150 sony\n",
      "713 It has title of DSC01151 sony\n",
      "715 It has title of DSC01154 sony\n",
      "717 It has title of DSC01155 sony\n",
      "719 It has title of DSC01156 sony\n",
      "721 It has title of DSC01158 sony\n",
      "723 It has title of DSC01159 sony\n",
      "725 It has title of DSC01160 sony\n",
      "727 It has title of DSC01161 sony\n",
      "729 It has title of DSC01162 sony\n",
      "731 It has title of DSC01163 sony\n",
      "733 It has title of DSC01164 sony\n",
      "735 It has title of DSC01165 sony\n",
      "737 It has title of DSC01167 sony\n",
      "739 It has title of DSC01100 sony\n",
      "740 It has title of DSC01168 sony\n",
      "742 It has title of IMG_0623 sony\n",
      "743 It has title of IMG_0624 sony\n",
      "744 It has title of IMG_0629 sony\n",
      "745 It has title of IMG_0630 sony\n",
      "746 It has title of IMG_2181 sony\n",
      "747 It has title of IMG_2182 sony\n",
      "748 It has title of IMG_6884 sony\n",
      "749 It has title of IMG_6885 sony\n",
      "750 It has title of IMG_6886 sony\n",
      "751 It has title of IMG_6887 sony\n",
      "752 It has title of IMG_6888 sony\n",
      "753 It has title of IMG_6889 sony\n",
      "755 It has title of IMG_0632 sony\n",
      "756 It has title of IMG_0634 sony\n",
      "757 It has title of IMG_0635 sony\n",
      "758 It has title of IMG_0648 sony\n",
      "759 It has title of IMG_2127 sony\n",
      "760 It has title of IMG_2128 sony\n",
      "761 It has title of IMG_2129 sony\n",
      "762 It has title of IMG_2130 sony\n",
      "763 It has title of IMG_1811 sony\n",
      "765 It has title of IMG_0633 sony\n",
      "766 It has title of IMG_0636 sony\n",
      "767 It has title of IMG_0637 sony\n",
      "768 It has title of IMG_0517 sony\n",
      "769 It has title of IMG_0518 sony\n",
      "770 It has title of IMG_0473 sony\n",
      "771 It has title of IMG_0474 sony\n",
      "772 It has title of DSC_20180421_10756 sony\n",
      "774 It has title of DSC_20180421_10757 sony\n",
      "776 It has title of DSC_20180421_10760 sony\n",
      "778 It has title of DSC_20180421_10768 sony\n",
      "779 It has title of IMG_1810 sony\n",
      "780 It has title of IMG_5026 sony\n",
      "781 It has title of IMG_5025 sony\n",
      "782 It has title of IMG_5024 sony\n",
      "783 It has title of IMG_5016 sony\n",
      "784 It has title of IMG_4805 sony\n",
      "785 It has title of IMG_4798 sony\n",
      "786 It has title of IMG_4785 sony\n",
      "787 It has title of IMG_4783 sony\n",
      "788 It has title of DSC_4087 sony\n",
      "789 It has title of DSC_4085 sony\n",
      "790 It has title of IMG_9476 sony\n",
      "791 It has title of IMG_2119 sony\n",
      "792 It has title of IMG_2120 sony\n",
      "793 It has title of IMG_2121 sony\n",
      "794 It has title of IMG_2125 sony\n",
      "795 It has title of IMG_9478 sony\n",
      "796 It has title of IMG_9480 sony\n",
      "797 It has title of IMG_9481 sony\n",
      "798 It has title of IMG_9482 sony\n",
      "799 It has title of IMG_9483 sony\n",
      "800 It has title of IMG_9484 sony\n",
      "801 It has title of IMG_9485 sony\n",
      "802 It has title of IMG_9486 sony\n",
      "803 It has title of IMG_9489 sony\n",
      "804 It has title of IMG_9490 sony\n",
      "805 It has title of IMG_9649 sony\n",
      "806 It has title of IMG_9650 sony\n",
      "807 It has title of IMG_9651 sony\n",
      "808 It has title of IMG_9652 sony\n",
      "809 It has title of IMG_9653 sony\n",
      "810 It has title of IMG_9654 sony\n",
      "811 It has title of IMG_9655 sony\n",
      "812 It has title of IMG_9656 sony\n",
      "813 It has title of IMG_9657 sony\n",
      "814 It has title of IMG_9658 sony\n",
      "815 It has title of IMG_9659 sony\n",
      "816 It has title of IMG_9660 sony\n",
      "817 It has title of IMG_9661 sony\n",
      "818 It has title of IMG_9662 sony\n",
      "819 It has title of IMG_9663 sony\n",
      "820 It has title of IMG_9665 sony\n",
      "821 It has title of IMG_9666 sony\n",
      "822 It has title of IMG_9667 sony\n",
      "823 It has title of IMG_9670 sony\n",
      "824 It has title of IMG_9673 sony\n",
      "825 It has title of IMG_9674 sony\n",
      "826 It has title of IMG_9675 sony\n",
      "827 It has title of IMG_9676 sony\n",
      "828 It has title of IMG_2122 sony\n",
      "829 It has title of IMG_2124 sony\n",
      "830 It has title of IMG_9678 sony\n",
      "831 It has title of IMG_9679 sony\n",
      "832 It has title of IMG_9680 sony\n",
      "833 It has title of IMG_9681 sony\n",
      "834 It has title of IMG_9686 sony\n",
      "835 It has title of IMG_9687 sony\n",
      "836 It has title of IMG_9688 sony\n",
      "837 It has title of IMG_9689 sony\n",
      "838 It has title of IMG_9321 sony\n",
      "839 It has title of IMG_9322 sony\n",
      "840 It has title of IMG_9777 sony\n",
      "841 It has title of IMG_9748 sony\n",
      "842 It has title of IMG_9749 sony\n",
      "843 It has title of IMG_9750 sony\n",
      "844 It has title of IMG_9751 sony\n",
      "845 It has title of IMG_9752 sony\n",
      "846 It has title of IMG_9753 sony\n",
      "847 It has title of IMG_9754 sony\n",
      "848 It has title of IMG_9755 sony\n",
      "849 It has title of IMG_9756 sony\n",
      "850 It has title of IMG_9757 sony\n",
      "851 It has title of IMG_9768 sony\n",
      "852 It has title of IMG_9769 sony\n",
      "853 It has title of IMG_9770 sony\n",
      "854 It has title of IMG_9771 sony\n",
      "855 It has title of IMG_9782 sony\n",
      "856 It has title of IMG_9783 sony\n",
      "857 It has title of IMG_9784 sony\n",
      "858 It has title of IMG_9785 sony\n",
      "859 It has title of IMG_9786 sony\n",
      "860 It has title of IMG_9787 sony\n",
      "861 It has title of IMG_9788 sony\n",
      "862 It has title of IMG_9789 sony\n",
      "863 It has title of IMG_9802 sony\n",
      "864 It has title of IMG_9803 sony\n",
      "865 It has title of IMG_9804 sony\n",
      "866 It has title of IMG_9805 sony\n",
      "867 It has title of IMG_9806 sony\n",
      "868 It has title of IMG_9807 sony\n",
      "869 It has title of IMG_9808 sony\n",
      "870 It has title of IMG_9809 sony\n",
      "871 It has title of IMG_9843 sony\n",
      "872 It has title of IMG_9844 sony\n",
      "873 It has title of IMG_9845 sony\n",
      "874 It has title of IMG_9846 sony\n",
      "875 It has title of IMG_9536 sony\n",
      "876 It has title of IMG_9537 sony\n",
      "877 It has title of IMG_9547 sony\n",
      "878 It has title of IMG_9548 sony\n",
      "879 It has title of IMG_9868 sony\n",
      "880 It has title of IMG_9869 sony\n",
      "881 It has title of IMG_9918 sony\n",
      "882 It has title of IMG_9919 sony\n",
      "883 It has title of IMG_9920 sony\n",
      "884 It has title of IMG_9921 sony\n",
      "885 It has title of IMG_9922 sony\n",
      "886 It has title of IMG_9821 sony\n",
      "887 It has title of IMG_9538 sony\n",
      "888 It has title of IMG_9539 sony\n",
      "889 It has title of IMG_9393 sony\n",
      "891 It has title of IMG_0184 sony\n",
      "892 It has title of IMG_0185 sony\n",
      "893 It has title of IMG_0147 sony\n",
      "894 It has title of IMG_0148 sony\n",
      "895 It has title of IMG_0140 sony\n",
      "896 It has title of IMG_0149 sony\n",
      "897 It has title of IMG_0151 sony\n",
      "898 It has title of IMG_0152 sony\n",
      "899 It has title of IMG_0223 sony\n",
      "900 It has title of IMG_0224 sony\n",
      "901 It has title of IMG_0379 sony\n",
      "902 It has title of IMG_0380 sony\n",
      "903 It has title of IMG_0381 sony\n",
      "904 It has title of IMG_0382 sony\n",
      "905 It has title of IMG_0477 sony\n",
      "906 It has title of IMG_0478 sony\n",
      "907 It has title of IMG_0488 sony\n",
      "908 It has title of IMG_0489 sony\n",
      "909 It has title of IMG_0540 sony\n",
      "910 It has title of IMG_0541 sony\n",
      "911 It has title of IMG_0546 sony\n",
      "912 It has title of IMG_0547 sony\n",
      "913 It has title of IMG_0577 sony\n",
      "914 It has title of IMG_0578 sony\n",
      "915 It has title of IMG_0579 sony\n",
      "916 It has title of IMG_0580 sony\n",
      "917 It has title of IMG_9772 sony\n",
      "918 It has title of IMG_9773 sony\n",
      "919 It has title of IMG_9774 sony\n",
      "920 It has title of IMG_9775 sony\n",
      "921 It has title of IMG_9776 sony\n",
      "922 It has title of IMG_9923 sony\n",
      "923 It has title of IMG_9822 sony\n",
      "924 It has title of IMG_9475 sony\n",
      "925 It has title of IMG_2126 sony\n",
      "926 It has title of IMG_9477 sony\n",
      "927 It has title of IMG_9479 sony\n",
      "928 It has title of IMG_9487 sony\n",
      "929 It has title of IMG_9488 sony\n",
      "935 OLYMPUS DIGITAL CAMERA sony\n",
      "936 OLYMPUS DIGITAL CAMERA sony\n",
      "937 OLYMPUS DIGITAL CAMERA sony\n",
      "938 OLYMPUS DIGITAL CAMERA sony\n",
      "939 OLYMPUS DIGITAL CAMERA sony\n",
      "942 OLYMPUS DIGITAL CAMERA sony\n",
      "943 OLYMPUS DIGITAL CAMERA sony\n",
      "944 It has title of 201612_ChangShuSuZhou_Mi4c_4 sony\n",
      "945 It has title of 201612_ChangShuSuZhou_Mi4c_8 sony\n",
      "946 It has title of 201612_ChangShuSuZhou_Mi4c_7 sony\n",
      "947 It has title of 201612_ChangShuSuZhou_Mi4c_9 sony\n",
      "948 It has title of 201612_ChangShuSuZhou_Mi4c_10 sony\n",
      "949 It has title of 201612_ChangShuSuZhou_Mi4c_17 sony\n",
      "950 It has title of DPP_9318 sony\n",
      "952 It has title of DPP_9305 sony\n",
      "954 It has title of DPP_9277 sony\n",
      "956 It has title of DPP_9274 sony\n",
      "958 It has title of DPP_9265 sony\n",
      "960 It has title of DPP_9261 sony\n",
      "962 It has title of DPP_9243 sony\n",
      "964 It has title of DPP_9237 sony\n",
      "966 It has title of DPP_9196 sony\n",
      "968 It has title of DPP_9204 sony\n",
      "970 It has title of DPP_9220 sony\n",
      "972 It has title of DPP_9222 sony\n",
      "974 It has title of DPP_9217 sony\n",
      "976 It has title of DPP_9227 sony\n",
      "978 It has title of IMG_4883 sony\n",
      "979 It has title of IMG_4881 sony\n",
      "980 It has title of IMG_4882 sony\n",
      "981 It has title of IMG_4276 sony\n",
      "982 It has title of IMG_4289 sony\n",
      "983 It has title of IMG_4290 sony\n",
      "984 It has title of IMG_4291 sony\n",
      "985 It has title of IMG_4292 sony\n",
      "986 It has title of IMG_4298 sony\n",
      "987 It has title of IMG_4300 sony\n",
      "988 It has title of IMG_4243 sony\n",
      "989 It has title of IMG_4301 sony\n",
      "990 It has title of IMG_4302 sony\n",
      "991 It has title of IMG_4305 sony\n",
      "992 It has title of IMG_4307 sony\n",
      "993 It has title of IMG_4311 sony\n",
      "994 It has title of IMG_4313 sony\n",
      "995 It has title of IMG_4318 sony\n",
      "996 It has title of IMG_4320 sony\n",
      "997 It has title of IMG_4319 sony\n",
      "998 It has title of IMG_4322 sony\n",
      "999 It has title of IMG_4324 sony\n",
      "1000 It has title of IMG_4326 sony\n",
      "1001 It has title of IMG_4328 sony\n",
      "1002 It has title of IMG_4332 sony\n",
      "1003 It has title of IMG_4334 sony\n",
      "1004 It has title of IMG_4335 sony\n",
      "1005 It has title of IMG_4337 sony\n",
      "1006 It has title of IMG_4338 sony\n",
      "1007 It has title of IMG_4341 sony\n",
      "1008 It has title of IMG_4342 sony\n",
      "1009 It has title of IMG_4344 sony\n",
      "1010 It has title of IMG_4345 sony\n",
      "1011 It has title of IMG_4346 sony\n",
      "1012 It has title of IMG_4348 sony\n",
      "1013 It has title of IMG_4350 sony\n",
      "1014 It has title of IMG_4354 sony\n",
      "1015 It has title of IMG_4355 sony\n",
      "1016 It has title of IMG_4357 sony\n",
      "1017 It has title of IMG_4356 sony\n",
      "1018 It has title of IMG_4263 sony\n",
      "1019 It has title of IMG_4268 sony\n",
      "1020 It has title of IMG_4273 sony\n",
      "1021 It has title of IMG_4275 sony\n",
      "1022 It has title of IMG_4287 sony\n",
      "1023 It has title of IMG_4283 sony\n",
      "1024 It has title of IMG_4294 sony\n",
      "1029 It has title of 20171112_IMG_0454 sony\n",
      "1030 It has title of 20171112_IMG_0455 sony\n",
      "1031 It has title of 20171112_IMG_0456 sony\n",
      "1032 It has title of 20171112_IMG_0463 sony\n",
      "1033 It has title of 20171112_IMG_0464 sony\n",
      "1034 It has title of 20171112_IMG_0465 sony\n",
      "1035 It has title of 20171112_IMG_0466 sony\n",
      "1036 It has title of 20171112_IMG_0467 sony\n",
      "1037 It has title of 20171112_IMG_0468 sony\n",
      "1038 It has title of 20171112_IMG_0469 sony\n",
      "1039 It has title of 20171112_IMG_0470 sony\n",
      "1040 It has title of 20171112_IMG_0473 sony\n",
      "1041 It has title of 20171112_IMG_0474 sony\n",
      "1042 It has title of 20171112_IMG_0475 sony\n",
      "1043 It has title of 20171112_IMG_0476 sony\n",
      "1044 It has title of 20171112_IMG_0477 sony\n",
      "1045 It has title of 20171112_IMG_0479 sony\n",
      "1046 It has title of 20171112_IMG_0480 sony\n",
      "1047 It has title of 20171112_IMG_0481 sony\n",
      "1048 It has title of 20171112_IMG_0482 sony\n",
      "1049 It has title of 20171112_IMG_0483 sony\n",
      "1050 It has title of 20171112_IMG_0484 sony\n",
      "1051 It has title of 20171112_IMG_0485 sony\n",
      "1052 It has title of 20171112_IMG_0486 sony\n",
      "1053 It has title of 20171112_IMG_0488 sony\n",
      "1054 It has title of 20171112_IMG_0489 sony\n",
      "1055 It has title of 20171112_IMG_0490 sony\n",
      "1056 It has title of 20171112_IMG_0491 sony\n",
      "1057 It has title of 20171112_IMG_0492 sony\n",
      "1058 It has title of 20171112_IMG_0493 sony\n",
      "1059 It has title of 20171112_IMG_0494 sony\n",
      "1060 It has title of 20171112_IMG_0495 sony\n",
      "1061 It has title of 20171112_IMG_0496 sony\n",
      "1062 It has title of 20171112_IMG_0498 sony\n",
      "1063 It has title of 20171112_IMG_0499 sony\n",
      "1064 It has title of 20171112_IMG_0500 sony\n",
      "1065 It has title of 20171112_IMG_0501 sony\n",
      "1066 It has title of 20171112_IMG_0457 sony\n",
      "1067 It has title of 20171112_IMG_0516 sony\n",
      "1068 It has title of 20171112_IMG_0517 sony\n",
      "1069 It has title of 20171112_IMG_0520 sony\n",
      "1070 It has title of 20171112_IMG_0521 sony\n",
      "1071 It has title of 20171112_IMG_0522 sony\n",
      "1072 It has title of 20171112_IMG_0523 sony\n",
      "1073 It has title of 20171112_IMG_0524 sony\n",
      "1074 It has title of 20171112_IMG_0525 sony\n",
      "1075 It has title of 20171112_IMG_0526 sony\n",
      "1076 It has title of 20171112_IMG_0527 sony\n",
      "1077 It has title of 20171112_IMG_0528 sony\n",
      "1078 It has title of 20171112_IMG_0529 sony\n",
      "1079 It has title of 20171112_IMG_0530 sony\n",
      "1080 It has title of 20171112_IMG_0532 sony\n",
      "1081 It has title of 20171112_IMG_0533 sony\n",
      "1082 It has title of 20171112_IMG_0534 sony\n",
      "1083 It has title of 20171112_IMG_0535 sony\n",
      "1084 It has title of 20171112_IMG_0536 sony\n",
      "1085 It has title of 20171112_IMG_0537 sony\n",
      "1086 It has title of 20171112_IMG_0538 sony\n",
      "1087 It has title of HOW_6132 sony\n",
      "1088 It has title of HOW_6135 sony\n",
      "1089 It has title of HOW_6024 sony\n",
      "1090 It has title of HOW_6027 sony\n",
      "1091 It has title of HOW_6037 sony\n",
      "1092 It has title of HOW_6026 sony\n",
      "1093 It has title of HOW_6039 sony\n",
      "1094 It has title of HOW_6041 sony\n",
      "1095 It has title of HOW_6043 sony\n",
      "1096 It has title of HOW_6053 sony\n",
      "1097 It has title of 66584-Suzhou sony\n",
      "1098 It has title of 66422-Suzhou sony\n",
      "1100 It has title of 66425-Suzhou sony\n",
      "1102 It has title of 66429-Suzhou sony\n",
      "1103 It has title of 66420-Suzhou sony\n",
      "1105 It has title of 66418-Suzhou sony\n",
      "1107 It has title of 66520-Suzhou sony\n",
      "1108 It has title of 66768-Suzhou sony\n",
      "1109 It has title of 66765-Suzhou sony\n",
      "1110 It has title of 66750-Suzhou sony\n",
      "1111 It has title of 66749-Suzhou sony\n",
      "1112 It has title of 66745-Suzhou sony\n",
      "1113 It has title of 66547-Suzhou sony\n",
      "1114 It has title of 66541-Suzhou sony\n",
      "1115 It has title of 66544-Suzhou sony\n",
      "1116 It has title of 66528-Suzhou sony\n",
      "1117 It has title of 66524-Suzhou sony\n",
      "1118 It has title of 66527-Suzhou sony\n",
      "1119 It has title of 66640-Suzhou sony\n",
      "1120 It has title of 66637-Suzhou sony\n",
      "1121 It has title of 66635-Suzhou sony\n",
      "1122 It has title of 66627-Suzhou sony\n",
      "1123 It has title of 66623-Suzhou sony\n",
      "1124 It has title of 66613-Suzhou sony\n",
      "1125 It has title of 66605-Suzhou sony\n",
      "1126 It has title of 66600-Suzhou sony\n",
      "1127 It has title of 66595-Suzhou sony\n",
      "1128 It has title of 66593-Suzhou sony\n",
      "1129 It has title of 66574-Suzhou sony\n",
      "1130 It has title of 66557-Suzhou sony\n",
      "1131 It has title of 66764-Suzhou sony\n",
      "1132 It has title of 66757-Suzhou sony\n",
      "1133 It has title of 66755-Suzhou sony\n",
      "1134 It has title of 48204-Suzhou sony\n",
      "1135 It has title of 66337-Suzhou sony\n",
      "1136 It has title of 66660-Suzhou sony\n",
      "1138 It has title of 66752-Suzhou sony\n",
      "1139 It has title of 66339-Suzhou sony\n",
      "1140 It has title of 66342-Suzhou sony\n",
      "1141 It has title of 66336-Suzhou sony\n",
      "1142 It has title of 66295-Suzhou sony\n",
      "1143 It has title of 66292-Suzhou sony\n",
      "1144 It has title of 66683-Suzhou sony\n",
      "1146 It has title of 66689-Suzhou sony\n",
      "1148 It has title of 66638-Suzhou sony\n",
      "1149 It has title of 66616-Suzhou sony\n",
      "1150 It has title of 66596-Suzhou sony\n",
      "1151 It has title of 66526-Suzhou sony\n",
      "1152 It has title of 66517-Suzhou sony\n",
      "1153 It has title of 66415-Suzhou sony\n",
      "1155 It has title of 66421-Suzhou sony\n",
      "1157 It has title of 66445-Suzhou sony\n",
      "1158 It has title of 66379-Suzhou sony\n",
      "1160 It has title of 66243-Suzhou sony\n",
      "1161 It has title of 66405-Suzhou sony\n",
      "1163 It has title of 66551-Suzhou sony\n",
      "1164 It has title of 66552-Suzhou sony\n",
      "1165 It has title of 66550-Suzhou sony\n",
      "1166 It has title of 66540-Suzhou sony\n",
      "1167 It has title of 66542-Suzhou sony\n",
      "1168 It has title of 66610-Suzhou sony\n",
      "1169 It has title of 66581-Suzhou sony\n",
      "1170 It has title of 66562-Suzhou sony\n",
      "1171 It has title of 66695-Suzhou sony\n",
      "1175 It has title of IMG_1256 sony\n",
      "1176 It has title of IMG_1244 sony\n",
      "1177 It has title of 20190517_F0001: Plaques and calligraphy sony\n",
      "1178 It has title of IMG_8491 sony\n",
      "1179 It has title of IMG_8492 sony\n",
      "1180 It has title of IMG_4092 sony\n",
      "1181 It has title of IMG_4080 sony\n",
      "1182 It has title of IMG_3871 sony\n",
      "1183 It has title of IMG_7346 sony\n",
      "1184 It has title of IMG_8494 sony\n",
      "1185 It has title of IMG_7316 sony\n",
      "1186 It has title of IMG_7322 sony\n",
      "1187 It has title of IMG_7323 sony\n",
      "1188 It has title of IMG_7324 sony\n",
      "1189 It has title of IMG_7325 sony\n",
      "1190 It has title of IMG_7326 sony\n",
      "1191 It has title of IMG_7327 sony\n",
      "1192 It has title of IMG_7328 sony\n",
      "1193 It has title of IMG_7329 sony\n",
      "1194 It has title of IMG_7330 sony\n",
      "1195 It has title of IMG_7331 sony\n",
      "1197 It has title of IMG_4224 sony\n",
      "1200 It has title of IMG_4247 sony\n",
      "1202 It has title of IMG_2509 sony\n",
      "1203 It has title of IMG_2670 sony\n",
      "1204 It has title of IMG_2655 sony\n",
      "1205 It has title of IMG_2656 sony\n",
      "1206 It has title of IMG_2500 sony\n",
      "1207 It has title of IMG_2501 sony\n",
      "1208 It has title of IMG_2502 sony\n",
      "1209 It has title of IMG_2504 sony\n",
      "1210 It has title of IMG_2503 sony\n",
      "1211 It has title of IMG_2505 sony\n",
      "1212 It has title of IMG_2508 sony\n",
      "1213 It has title of IMG_2506 sony\n",
      "1214 It has title of IMG_2507 sony\n",
      "1215 It has title of IMG_2510 sony\n",
      "1216 It has title of IMG_2511 sony\n",
      "1217 It has title of IMG_2512 sony\n",
      "1218 It has title of IMG_2513 sony\n",
      "1219 It has title of IMG_2517 sony\n",
      "1220 It has title of IMG_6377 sony\n",
      "1221 It has title of IMG_6378 sony\n",
      "1222 It has title of IMG_2474 sony\n",
      "1223 It has title of IMG_2475 sony\n",
      "1224 It has title of IMG_2476 sony\n",
      "1225 It has title of IMG_2477 sony\n",
      "1226 It has title of IMG_6375 sony\n",
      "1227 It has title of IMG_6376 sony\n",
      "1228 It has title of IMG_6373 sony\n",
      "1229 It has title of IMG_2766 sony\n",
      "1230 It has title of IMG_2767 sony\n",
      "1231 It has title of IMG_2737 sony\n",
      "1232 It has title of IMG_2736 sony\n",
      "1233 It has title of IMG_2738 sony\n",
      "1234 It has title of IMG_2739 sony\n",
      "1235 It has title of IMG_2741 sony\n",
      "1236 It has title of IMG_2740 sony\n",
      "1237 It has title of IMG_2734 sony\n",
      "1238 It has title of IMG_2742 sony\n",
      "1239 It has title of IMG_2735 sony\n",
      "1240 It has title of IMG_2743 sony\n",
      "1241 It has title of IMG_2744 sony\n",
      "1242 It has title of IMG_2745 sony\n",
      "1243 It has title of IMG_2746 sony\n",
      "1244 It has title of IMG_1113 sony\n",
      "1246 It has title of IMG_1115 sony\n",
      "1248 It has title of IMG_1116 sony\n",
      "1274 It has title of IMG_2875 sony\n",
      "1275 It has title of IMG_2876 sony\n",
      "1276 It has title of DSC01091 sony\n",
      "1278 It has title of DSC01092 sony\n",
      "1279 It has title of DSC01096 sony\n",
      "1280 It has title of DSC01098 sony\n",
      "1281 It has title of DSC01104 sony\n",
      "1283 It has title of DSC01108 sony\n",
      "1285 It has title of DSC01109 sony\n",
      "1287 It has title of DSC01112 sony\n",
      "1288 It has title of DSC01113 sony\n",
      "1290 It has title of DSC01114 sony\n",
      "1291 It has title of DSC01115 sony\n",
      "1293 It has title of DSC01116 sony\n",
      "1294 It has title of DSC01117 sony\n",
      "1295 It has title of DSC01118 sony\n",
      "1296 It has title of DSC01119 sony\n",
      "1298 It has title of DSC01120 sony\n",
      "1300 It has title of DSC01121 sony\n",
      "1301 It has title of DSC01123 sony\n",
      "1302 It has title of DSC01124a sony\n",
      "1303 It has title of DSC01128 sony\n",
      "1305 It has title of DSC01134 sony\n",
      "1307 It has title of DSC01135 sony\n",
      "1309 It has title of DSC01137 sony\n",
      "1311 It has title of DSC01141 sony\n",
      "1313 It has title of DSC01157 sony\n",
      "1315 It has title of IMG_2881 sony\n",
      "1316 It has title of IMG_2861 sony\n",
      "1317 It has title of IMG_1637 sony\n",
      "1318 It has title of IMG20180622110713 sony\n",
      "1319 It has title of IMG20180622112358 sony\n",
      "1320 It has title of IMG20180622112604 sony\n",
      "1321 It has title of DSC_20180421_10779 sony\n",
      "1323 It has title of DSC_20180421_10781 sony\n",
      "1325 It has title of DSC01094 sony\n",
      "1326 It has title of DSC01095 sony\n",
      "1328 It has title of DSC01099 sony\n",
      "1330 It has title of DSC01126 sony\n",
      "1332 It has title of DSC01127 sony\n",
      "1334 It has title of DSC01138 sony\n",
      "1336 It has title of IMG20180622112538 sony\n",
      "1337 It has title of IMG20180622112533 sony\n",
      "1338 It has title of IMG20180622111549 sony\n",
      "1339 It has title of IMG20180622111435 sony\n",
      "1340 It has title of IMG20180622112337 sony\n",
      "1341 It has title of IMG20180622112201 sony\n",
      "1342 It has title of IMG20180622111511 sony\n",
      "1343 It has title of IMG20180622154431 sony\n",
      "1344 It has title of IMG20180622112649 sony\n",
      "1345 It has title of IMG20180622112010 sony\n",
      "1346 It has title of IMG20180622112732 sony\n",
      "1347 It has title of IMG20180622153648 sony\n",
      "1348 It has title of IMG20180622112644 sony\n",
      "1349 It has title of IMG20180622111408 sony\n",
      "1350 It has title of IMG20180622112611 sony\n",
      "1351 It has title of IMG20180622112118 sony\n",
      "1352 It has title of IMG_5677 sony\n",
      "1353 It has title of IMG_5678 sony\n",
      "1354 It has title of IMG_5093 sony\n",
      "1355 It has title of IMG_5094 sony\n",
      "1356 It has title of IMG_4698 sony\n",
      "1357 It has title of IMG_4710 sony\n",
      "1358 It has title of IMG_4699 sony\n",
      "1359 It has title of IMG_4700 sony\n",
      "1360 It has title of IMG_4701 sony\n",
      "1361 It has title of IMG_2988 sony\n",
      "1362 It has title of IMG_2989 sony\n",
      "1363 It has title of IMG_2927 sony\n",
      "1364 It has title of IMG_2928 sony\n",
      "1365 It has title of IMG_2929 sony\n",
      "1366 It has title of IMG_2932 sony\n",
      "1367 It has title of IMG_2933 sony\n",
      "1368 It has title of IMG_2187 sony\n",
      "1369 It has title of IMG_2188 sony\n",
      "1370 It has title of IMG_2189 sony\n",
      "1371 It has title of IMG_2190 sony\n",
      "1372 It has title of IMG_2152 sony\n",
      "1373 It has title of IMG_2153 sony\n",
      "1374 It has title of IMG_2154 sony\n",
      "1375 It has title of IMG_2155 sony\n",
      "1376 It has title of IMG_2156 sony\n",
      "1381 It has title of IMG_0245 sony\n",
      "1382 It has title of DPP_9232 sony\n",
      "1384 It has title of DPP_9233 sony\n",
      "1386 It has title of IMG_0296 sony\n",
      "1387 It has title of IMG_0294 sony\n",
      "1388 It has title of IMG_0290 sony\n",
      "1389 It has title of IMG_0286 sony\n",
      "1390 It has title of IMG_0283 sony\n",
      "1391 It has title of IMG_0276 sony\n",
      "1392 It has title of IMG_0258 sony\n",
      "1393 It has title of IMG_0423 sony\n",
      "1394 It has title of IMG_0537 sony\n",
      "1395 It has title of IMG_0536 sony\n",
      "1396 It has title of IMG_0538 sony\n",
      "1397 It has title of IMG_5859 sony\n",
      "1398 It has title of IMG_0247 sony\n",
      "1399 It has title of IMG_0241 sony\n",
      "1400 It has title of IMG_0233 sony\n",
      "1401 It has title of IMG_0237 sony\n",
      "1402 It has title of IMG_0225 sony\n",
      "1413 It has title of IMG_9664 sony\n",
      "1414 It has title of IMG_9668 sony\n",
      "1415 It has title of IMG_9669 sony\n",
      "1416 It has title of IMG_2123 sony\n",
      "1417 It has title of IMG_9327 sony\n",
      "1418 It has title of IMG_9328 sony\n",
      "1419 It has title of IMG_9329 sony\n",
      "1420 It has title of IMG_9331 sony\n",
      "1421 It has title of IMG_9332 sony\n",
      "1422 It has title of IMG_9335 sony\n",
      "1423 It has title of IMG_9463 sony\n",
      "1424 It has title of IMG_9472 sony\n",
      "1425 It has title of IMG_9473 sony\n",
      "1426 It has title of IMG_9474 sony\n",
      "1427 It has title of IMG_9724 sony\n",
      "1428 It has title of IMG_9725 sony\n",
      "1429 It has title of IMG_9323 sony\n",
      "1430 It has title of IMG_9324 sony\n",
      "1431 It has title of IMG_9344 sony\n",
      "1432 It has title of IMG_9345 sony\n",
      "1433 It has title of IMG_9352 sony\n",
      "1434 It has title of IMG_9353 sony\n",
      "1435 It has title of IMG_9459 sony\n",
      "1436 It has title of IMG_9460 sony\n",
      "1437 It has title of IMG_9461 sony\n",
      "1438 It has title of IMG_9462 sony\n",
      "1439 It has title of IMG_9464 sony\n",
      "1440 It has title of IMG_0272 sony\n",
      "1441 It has title of IMG_0539 sony\n",
      "1459 It has title of IMG_0424 sony\n",
      "1460 It has title of DSC21244 sony\n",
      "1462 It has title of IMG_6977 sony\n",
      "1464 It has title of IMG_6983 sony\n",
      "1466 It has title of IMG_6987 sony\n",
      "1468 It has title of IMG_6892 sony\n",
      "1470 It has title of IMG_6894 sony\n",
      "1472 It has title of IMG_6899-2 sony\n",
      "1474 It has title of IMG_6899 sony\n",
      "1476 It has title of IMG_6909 sony\n",
      "1478 It has title of IMG_6912 sony\n",
      "1480 It has title of IMG_6930 sony\n",
      "1482 It has title of IMG_6934 sony\n",
      "1484 It has title of IMG_6938 sony\n",
      "1486 It has title of IMG_6953 sony\n",
      "1488 It has title of IMG_6954 sony\n",
      "1490 It has title of IMG_0911 sony\n",
      "1491 It has title of IMG_20200510_152440 sony\n",
      "1492 It has title of HOW_6060 sony\n",
      "1493 It has title of 66261-Suzhou sony\n",
      "1494 It has title of 66256-Suzhou sony\n",
      "1495 It has title of 66247-Suzhou sony\n",
      "1496 It has title of 66245-Suzhou sony\n",
      "1497 It has title of 66226-Suzhou sony\n",
      "1498 It has title of 66228-Suzhou sony\n",
      "1499 It has title of 66233-Suzhou sony\n",
      "1500 It has title of 66229-Suzhou sony\n",
      "1501 It has title of 66357-Suzhou sony\n",
      "1502 It has title of 66353-Suzhou sony\n",
      "1503 It has title of 66345-Suzhou sony\n",
      "1504 It has title of 66340-Suzhou sony\n",
      "1505 It has title of 48348-Suzhou sony\n",
      "1506 It has title of 66549-Suzhou sony\n",
      "1507 It has title of 48506-Suzhou sony\n",
      "1508 It has title of IMG_1282 sony\n",
      "1509 It has title of IMG_1254 sony\n",
      "1511 It has title of IMG_4248 sony\n",
      "1514 It has title of IMG_4253 sony\n",
      "1516 It has title of IMG_20190319_144335_BURST002 sony\n",
      "1517 It has title of IMG_20190319_161314_BURST002 sony\n",
      "1518 It has title of IMG_20190319_161314_BURST003 sony\n",
      "1519 It has title of IMG_20190319_161314_BURST004 sony\n",
      "1520 It has title of IMG_20190319_161314_BURST005 sony\n",
      "1521 It has title of IMG_20190319_161547 sony\n",
      "1522 It has title of IMG_20190319_161540_BURST002 sony\n",
      "1523 It has title of IMG_2755 sony\n",
      "1524 It has title of IMG_2751 sony\n",
      "1525 It has title of IMG_2752 sony\n",
      "1526 It has title of IMG_2753 sony\n",
      "1527 It has title of IMG_2754 sony\n",
      "1528 It has title of IMG_2756 sony\n",
      "1529 It has title of IMG_2757 sony\n",
      "1530 It has title of IMG_2758 sony\n",
      "1531 It has title of IMG_2759 sony\n",
      "1532 It has title of IMG_2760 sony\n",
      "1533 It has title of IMG_2761 sony\n",
      "1534 It has title of IMG_2762 sony\n",
      "1535 It has title of IMG_2721 sony\n",
      "1536 It has title of IMG_2729 sony\n",
      "1537 It has title of IMG_2722 sony\n",
      "1538 It has title of IMG_2723 sony\n",
      "1539 It has title of IMG_2724 sony\n",
      "1540 It has title of IMG_2730 sony\n",
      "1541 It has title of IMG_2717 sony\n",
      "1542 It has title of IMG_2718 sony\n",
      "1543 It has title of IMG_2665 sony\n",
      "1544 It has title of IMG_2666 sony\n",
      "1545 It has title of IMG_2667 sony\n",
      "1546 It has title of IMG_2668 sony\n",
      "1547 It has title of IMG_2669 sony\n",
      "1548 It has title of IMG_2479 sony\n",
      "1549 It has title of IMG_1108 sony\n",
      "1552 It has title of IMG_5046 sony\n",
      "1553 It has title of IMG_5047 sony\n",
      "1561 It has title of IMG20180622112443 sony\n",
      "1562 It has title of IMG20180622112017 sony\n",
      "1563 It has title of IMG20180622112106 sony\n",
      "1564 It has title of IMG20180622154514 sony\n",
      "1565 It has title of IMG20180622111514 sony\n",
      "1566 It has title of IMG20180622150117 sony\n",
      "1567 It has title of IMG20180622111553 sony\n",
      "1568 It has title of IMG20180622110739 sony\n",
      "1569 It has title of IMG20180622112448 sony\n",
      "1570 It has title of IMG20180622153616 sony\n",
      "1571 It has title of DSC_20180421_10839 sony\n",
      "1572 It has title of DSC_20180421_10842 sony\n",
      "1574 It has title of DSC_20180421_10850 sony\n",
      "1575 It has title of DSC_20180421_10851 sony\n",
      "1577 It has title of DSC_20180421_10856 sony\n",
      "1579 It has title of DSC_20180421_10865 sony\n",
      "1581 It has title of DSC_20180421_10873 sony\n",
      "1583 It has title of DSC_20180421_10878 sony\n",
      "1585 It has title of DSC_20180421_10879 sony\n",
      "1587 It has title of DSC_20180421_10882 sony\n",
      "1589 It has title of DSC_20180421_10886 sony\n",
      "1591 It has title of DSC_20180421_10889 sony\n",
      "1593 It has title of DSC_20180421_10891 sony\n",
      "1595 It has title of DSC_20180421_10899 sony\n",
      "1597 It has title of DSC_20180421_10900 sony\n",
      "1599 It has title of DSC_20180421_10907 sony\n",
      "1601 It has title of DSC_20180421_10910 sony\n",
      "1603 It has title of DSC_20180421_10912 sony\n",
      "1605 It has title of DSC_20180421_10914 sony\n",
      "1607 It has title of DSC_20180421_10918 sony\n",
      "1609 It has title of DSC_20180421_10921 sony\n",
      "1611 It has title of DSC_20180421_10924 sony\n",
      "1613 It has title of DSC_20180421_10926 sony\n",
      "1615 It has title of DSC_20180421_10928 sony\n",
      "1617 It has title of DSC_20180421_10929 sony\n",
      "1619 It has title of DSC_20180421_10931 sony\n",
      "1621 It has title of DSC_20180421_10933 sony\n",
      "1623 It has title of DSC_20180421_10938 sony\n",
      "1625 It has title of DSC_20180421_10944 sony\n",
      "1627 It has title of DSC_20180421_10947 sony\n",
      "1629 It has title of DSC_20180421_10951 sony\n",
      "1631 It has title of DSC_20180421_10956 sony\n",
      "1633 It has title of DSC_20180421_10964 sony\n",
      "1635 It has title of DSC_20180421_10970 sony\n",
      "1637 It has title of DSC_20180421_10971 sony\n",
      "1639 It has title of DSC_20180421_10972 sony\n",
      "1641 It has title of DSC_20180421_10973 sony\n",
      "1643 It has title of DSC_20180421_10769 sony\n",
      "1645 It has title of DSC_20180421_10969 sony\n",
      "1647 It has title of DSC_20180421_10975 sony\n",
      "1649 It has title of IMG_9465 sony\n",
      "1650 It has title of IMG_9466 sony\n",
      "1651 It has title of IMG_9468 sony\n",
      "1652 It has title of IMG_9470 sony\n",
      "1653 It has title of IMG_9471 sony\n",
      "1654 It has title of IMG_9723 sony\n",
      "1655 It has title of IMG_9726 sony\n",
      "1656 It has title of IMG_9727 sony\n",
      "1657 It has title of IMG_9729 sony\n",
      "1658 It has title of IMG_9730 sony\n",
      "1659 It has title of IMG_9731 sony\n",
      "1660 It has title of IMG_9732 sony\n",
      "1661 It has title of IMG_9733 sony\n",
      "1662 It has title of IMG_8018 sony\n",
      "1663 It has title of IMG_8019 sony\n",
      "1664 It has title of DSC_4079 sony\n",
      "1665 It has title of DSC_4050 sony\n",
      "1666 It has title of IMG_20180326_090901 sony\n",
      "1667 It has title of IMG_20180326_090933 sony\n",
      "1668 It has title of IMG_20180326_103116 sony\n",
      "1669 It has title of IMG_20180326_103533_HDR sony\n",
      "1672 It has title of 201612_ChangShuSuZhou_Mi4c_16 sony\n",
      "1673 It has title of IMG_4317 sony\n",
      "1674 It has title of IMG_4330 sony\n",
      "1681 It has title of 66419-Suzhou sony\n",
      "1683 It has title of 66413-Suzhou sony\n",
      "1684 It has title of 66414-Suzhou sony\n",
      "1686 It has title of 66283-Suzhou sony\n",
      "1687 It has title of 66278-Suzhou sony\n",
      "1688 It has title of 66264-Suzhou sony\n",
      "1689 It has title of 66289-Suzhou sony\n",
      "1690 It has title of 66285-Suzhou sony\n",
      "1691 It has title of IMG_9246 sony\n",
      "1692 It has title of IMG_9247 sony\n",
      "1693 It has title of IMG_9248 sony\n",
      "1694 It has title of IMG_9250 sony\n",
      "1695 It has title of IMG_9251 sony\n",
      "1696 It has title of IMG_9230 sony\n",
      "1697 It has title of IMG_9232 sony\n",
      "1698 It has title of IMG_9233 sony\n",
      "1699 It has title of IMG_8382 sony\n",
      "1700 It has title of IMG_4113 sony\n",
      "1701 It has title of IMG_4114 sony\n",
      "1702 It has title of IMG_8493 sony\n",
      "1703 It has title of IMG_2747 sony\n",
      "1704 It has title of IMG_2748 sony\n",
      "1705 It has title of IMG_2749 sony\n",
      "1706 It has title of IMG_2750 sony\n",
      "1707 It has title of IMG_2516 sony\n",
      "1708 It has title of IMG_2478 sony\n",
      "1709 It has title of IMG20180622110720 sony\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1710 It has title of IMG_2151 sony\n",
      "1711 It has title of DSC_20180421_10773 sony\n",
      "1713 It has title of DSC_20180421_10776 sony\n",
      "1715 It has title of DSC_20180421_10782 sony\n",
      "1717 It has title of DSC_20180421_10784 sony\n",
      "1719 It has title of DSC_20180421_10786 sony\n",
      "1721 It has title of DSC_20180421_10788 sony\n",
      "1723 It has title of DSC_20180421_10792 sony\n",
      "1725 It has title of DSC_20180421_10795 sony\n",
      "1727 It has title of DSC_20180421_10796 sony\n",
      "1729 It has title of DSC_20180421_10799 sony\n",
      "1731 It has title of DSC_20180421_10802 sony\n",
      "1733 It has title of IMG_20180326_104234 sony\n",
      "1734 It has title of IMG_20180326_110007 sony\n",
      "1735 It has title of IMG_20180326_110426 sony\n",
      "1736 It has title of IMG_20180326_110804 sony\n",
      "1737 It has title of IMG_20180326_103247 sony\n",
      "1738 It has title of IMG_20180326_105311 sony\n",
      "1739 It has title of IMG_0168 sony\n",
      "1740 It has title of IMG_0169 sony\n",
      "1741 It has title of DSC_20180421_10816 sony\n",
      "1743 It has title of DSC_20180421_10818 sony\n",
      "1745 It has title of DSC_20180421_10825 sony\n",
      "1746 It has title of DSC_20180421_10830 sony\n",
      "1747 It has title of DSC_20180421_10836 sony\n",
      "1749 It has title of IMG_0170 sony\n",
      "1750 It has title of IMG_9467 sony\n",
      "1751 It has title of IMG_9722 sony\n",
      "1752 It has title of IMG_9728 sony\n",
      "1753 It has title of IMG_20180326_103831 sony\n",
      "1754 It has title of IMG_20180326_103555_HDR sony\n",
      "1755 It has title of IMG_20180326_104102 sony\n",
      "1756 It has title of IMG_20180326_122350 sony\n",
      "1757 It has title of IMG_0167 sony\n",
      "1758 It has title of IMG_9469 sony\n",
      "1759 OLYMPUS DIGITAL CAMERA sony\n",
      "1760 It has title of IMG_6925 sony\n",
      "1762 It has title of IMG_6928 sony\n",
      "1765 It has title of IMG_5472 sony\n",
      "1766 It has title of IMG_5474 sony\n",
      "1767 It has title of IMG_5478 sony\n",
      "1768 It has title of IMG_5483 sony\n",
      "1769 It has title of IMG_5492 sony\n",
      "1770 It has title of IMG_5502 sony\n",
      "1771 It has title of IMG_5451 sony\n",
      "1772 It has title of IMG_5446 sony\n",
      "1775 It has title of 20171112_IMG_0502 sony\n",
      "1776 It has title of 20171113_IMG_0591 sony\n",
      "1777 It has title of 20171113_IMG_0592 sony\n",
      "1778 It has title of 20171113_IMG_0595 sony\n",
      "1779 It has title of 20171113_IMG_0596 sony\n",
      "1780 It has title of 20171113_IMG_0597 sony\n",
      "1781 It has title of 20171113_IMG_0598 sony\n",
      "1782 It has title of 20171113_IMG_0601 sony\n",
      "1783 It has title of 20171113_IMG_0602 sony\n",
      "1784 It has title of 20171113_IMG_0603 sony\n",
      "1785 It has title of 20171113_IMG_0607 sony\n",
      "1786 It has title of 20171113_IMG_0608 sony\n",
      "1787 It has title of 20171113_IMG_0609 sony\n",
      "1788 It has title of 20171113_IMG_0610 sony\n",
      "1789 It has title of 20171113_IMG_0611 sony\n",
      "1790 It has title of 20171113_IMG_0612 sony\n",
      "1791 It has title of 20171113_IMG_0613 sony\n",
      "1792 It has title of 20171113_IMG_0614 sony\n",
      "1793 It has title of 20171113_IMG_0615 sony\n",
      "1794 It has title of 20171113_IMG_0628 sony\n",
      "1795 It has title of 20171113_IMG_0639 sony\n",
      "1796 It has title of 20171113_IMG_0646 sony\n",
      "1797 It has title of 20171113_IMG_0647 sony\n",
      "1798 It has title of 20171113_IMG_0648 sony\n",
      "1799 It has title of 20171113_IMG_0649 sony\n",
      "1800 It has title of 20171113_IMG_0651 sony\n",
      "1801 It has title of 20171113_IMG_0652 sony\n",
      "1802 It has title of 20171113_IMG_0653 sony\n",
      "1803 It has title of 20171113_IMG_0654 sony\n",
      "1804 It has title of 20171113_IMG_0655 sony\n",
      "1805 It has title of 20171113_IMG_0656 sony\n",
      "1806 It has title of 20171113_IMG_0657 sony\n",
      "1807 It has title of 20171113_IMG_0658 sony\n",
      "1808 It has title of 20171113_IMG_0659 sony\n",
      "1809 It has title of 20171113_IMG_0661 sony\n",
      "1810 It has title of 20171113_IMG_0662 sony\n",
      "1811 It has title of 20171113_IMG_0664 sony\n",
      "1812 It has title of 20171113_IMG_0666 sony\n",
      "1813 It has title of 20171113_IMG_0665 sony\n",
      "1814 It has title of 20171113_IMG_0667 sony\n",
      "1815 It has title of 20171113_IMG_0670 sony\n",
      "1816 It has title of 20171113_IMG_0671 sony\n",
      "1817 It has title of 20171113_IMG_0672 sony\n",
      "1818 It has title of 20171113_IMG_0673 sony\n",
      "1819 It has title of 20171113_IMG_0674 sony\n",
      "1820 It has title of 20171113_IMG_0675 sony\n",
      "1821 It has title of 20171113_IMG_0676 sony\n",
      "1822 It has title of 20171113_IMG_0677 sony\n",
      "1823 It has title of 20171113_IMG_0678 sony\n",
      "1824 It has title of 20171113_IMG_0679 sony\n",
      "1825 It has title of 20171113_IMG_0663 sony\n",
      "1826 It has title of 20171112_IMG_0503 sony\n",
      "1827 It has title of 20171112_IMG_0504 sony\n",
      "1828 It has title of 20171112_IMG_0505 sony\n",
      "1829 It has title of 20171112_IMG_0506 sony\n",
      "1830 It has title of 20171112_IMG_0507 sony\n",
      "1831 It has title of 20171112_IMG_0508 sony\n",
      "1832 It has title of 20171112_IMG_0509 sony\n",
      "1833 It has title of 20171112_IMG_0510 sony\n",
      "1834 It has title of 20171112_IMG_0511 sony\n",
      "1835 It has title of 20171112_IMG_0512 sony\n",
      "1836 It has title of 20171112_IMG_0513 sony\n",
      "1837 It has title of 20171112_IMG_0514 sony\n",
      "1838 It has title of 20171112_IMG_0515 sony\n",
      "1839 It has title of 20171112_IMG_0518 sony\n",
      "1840 It has title of 20171112_IMG_0531 sony\n",
      "1841 It has title of 20171112_IMG_0539 sony\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(origindicts)):\n",
    "    dic1 = origindicts[i]\n",
    "    dic2 = targdicts[i]\n",
    "    for k,v in dic1.items():\n",
    "        record = 1\n",
    "        if v.startswith('It has tags of'):\n",
    "            dic2[k] = v\n",
    "        elif v.startswith('It has title of'):\n",
    "            for w in stoplist:\n",
    "                if ' '.join(v.lower().split()[4:]).startswith(w) or ' '.join(v.split()[4:])[:4].isupper() or ' '.join(v.split()[4:])[:4].isdigit():\n",
    "                    record = 0\n",
    "                    print(k,v,w)\n",
    "                    break\n",
    "            if record==1:\n",
    "                dic2[k] = v\n",
    "        else:\n",
    "            for w in stoplist:\n",
    "                if v.lower().startswith(w) or v[:5].isupper() or v[:5].isdigit():\n",
    "                    record = 0\n",
    "                    print(k,v,w)\n",
    "                    break\n",
    "            if record==1:\n",
    "                dic2[k] = v"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e07caee4",
   "metadata": {},
   "source": [
    "After the filtering, 754 out of 1963 data points remained as data with valid text description."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "fd0b4759",
   "metadata": {},
   "outputs": [],
   "source": [
    "filter_en = [origin_en[k] for k in filter_dict_en]\n",
    "filter_cn = [origin_cn[k] for k in filter_dict_cn]\n",
    "filter_other = [origin_other[k] for k in filter_dict_other]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "64bdb851",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(368, 27, 413, 754)"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(filter_en)), len(set(filter_cn)), len(set(filter_other)), len(set(filter_en).union(set(filter_cn)).union(set(filter_other)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "ce1279df",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 'It has title of G for Garden',\n",
       " 1: 'It has tags of china, china, chinese, garden, suzhou, chinese',\n",
       " 4: 'It has tags of garden',\n",
       " 5: 'It has tags of garden',\n",
       " 6: 'Laure and China and 2007',\n",
       " 7: 'It has title of Suzhou_01',\n",
       " 8: 'Laure and China and 2007',\n",
       " 9: 'It has title of Suzhou_02',\n",
       " 10: 'Laure and China and 2007',\n",
       " 11: 'It has title of Suzhou_03',\n",
       " 12: 'It has tags of dji, suzhou, jiangsu, peoplesrepublicofchina',\n",
       " 13: 'It has title of Canglang Pavilion',\n",
       " 14: 'It has tags of sony vsco',\n",
       " 15: 'It has tags of sony vsco',\n",
       " 16: 'It has title of Canglang Pavilion',\n",
       " 17: 'It has tags of sony, vsco, nature, architecture, design',\n",
       " 18: 'It has title of Suzhou',\n",
       " 19: 'It has title of Homemade',\n",
       " 20: 'It has title of Suzhou',\n",
       " 21: 'It has tags of china, chine, suzhou',\n",
       " 22: 'It has tags of china, suzhou',\n",
       " 23: 'It has title of Suzhou, Shan Tang st.',\n",
       " 24: 'It has tags of china, suzhou',\n",
       " 45: 'It has title of Suzhou 8',\n",
       " 58: 'Lion Grove Garden',\n",
       " 101: 'It has tags of suzhou, china, jiangsu',\n",
       " 102: 'It has tags of suzhou, china, jiangsu',\n",
       " 104: 'It has title of Mirror',\n",
       " 121: 'It has tags of arts, centre, china, culture, jiangsu, suzhou, peoplesrepublicofchina',\n",
       " 131: 'It has tags of china, jiangsu, jinji, lake, suzhou, peoplesrepublicofchina',\n",
       " 133: 'It has tags of china, jiangsu, jinji, lake, suzhou, peoplesrepublicofchina',\n",
       " 135: 'It has tags of china, jiangsu, jinji, lake, suzhou, peoplesrepublicofchina',\n",
       " 137: 'It has tags of china, jiangsu, jinji, lake, suzhou, peoplesrepublicofchina',\n",
       " 139: 'It has tags of china, jiangsu, jinji, lake, suzhou, peoplesrepublicofchina',\n",
       " 141: 'It has tags of arts, centre, china, culture, jiangsu, suzhou, peoplesrepublicofchina',\n",
       " 143: 'It has tags of arts, centre, china, culture, jiangsu, suzhou, peoplesrepublicofchina',\n",
       " 145: 'It has tags of arts, centre, china, culture, jiangsu, suzhou, peoplesrepublicofchina',\n",
       " 147: 'It has tags of arts, centre, china, culture, jiangsu, suzhou, peoplesrepublicofchina',\n",
       " 149: 'It has tags of arts, centre, china, culture, jiangsu, suzhou, peoplesrepublicofchina',\n",
       " 151: 'It has tags of arts, centre, china, culture, jiangsu, suzhou, peoplesrepublicofchina',\n",
       " 153: 'It has tags of arts, centre, china, culture, jiangsu, suzhou, peoplesrepublicofchina',\n",
       " 155: 'It has tags of arts, centre, china, culture, jiangsu, suzhou, peoplesrepublicofchina',\n",
       " 157: 'It has tags of arts, centre, china, culture, jiangsu, suzhou, peoplesrepublicofchina',\n",
       " 159: 'It has tags of arts, centre, china, culture, jiangsu, suzhou, peoplesrepublicofchina',\n",
       " 161: 'It has tags of arts, centre, china, culture, jiangsu, suzhou, peoplesrepublicofchina',\n",
       " 177: 'It has tags of china, jiangsu, pingjiang, suzhou, peoplesrepublicofchina',\n",
       " 184: 'It has tags of china, jiangsu, pingjiang, suzhou, peoplesrepublicofchina',\n",
       " 194: 'It has tags of center, central, china, jiangsu, plaza, square, suzhou, peoplesrepublicofchina',\n",
       " 196: 'It has tags of center, central, china, jiangsu, plaza, square, suzhou, peoplesrepublicofchina',\n",
       " 198: 'It has tags of center, central, china, jiangsu, plaza, square, suzhou, peoplesrepublicofchina',\n",
       " 201: 'It has tags of center, central, china, jiangsu, plaza, square, suzhou, peoplesrepublicofchina',\n",
       " 231: 'It has tags of china, jiangsu, jinji, lake, suzhou, peoplesrepublicofchina',\n",
       " 233: 'It has tags of china, jiangsu, jinji, lake, suzhou, peoplesrepublicofchina',\n",
       " 235: 'It has tags of china, jiangsu, jinji, lake, suzhou, peoplesrepublicofchina',\n",
       " 237: 'It has tags of arts, centre, china, culture, jiangsu, suzhou, peoplesrepublicofchina',\n",
       " 239: 'It has tags of china, jiangsu, jinji, lake, suzhou, peoplesrepublicofchina',\n",
       " 241: 'It has tags of arts, centre, china, culture, jiangsu, suzhou, peoplesrepublicofchina',\n",
       " 243: 'It has tags of arts, centre, china, culture, jiangsu, suzhou, peoplesrepublicofchina',\n",
       " 245: 'It has tags of arts, centre, china, culture, jiangsu, suzhou, peoplesrepublicofchina',\n",
       " 247: 'It has tags of arts, centre, china, culture, jiangsu, suzhou, peoplesrepublicofchina',\n",
       " 249: 'It has tags of arts, centre, china, culture, jiangsu, suzhou, peoplesrepublicofchina',\n",
       " 251: 'It has tags of arts, centre, china, culture, jiangsu, suzhou, peoplesrepublicofchina',\n",
       " 253: 'It has tags of arts, centre, china, culture, jiangsu, suzhou, peoplesrepublicofchina',\n",
       " 255: 'It has tags of arts, centre, china, culture, jiangsu, suzhou, peoplesrepublicofchina',\n",
       " 257: 'It has tags of arts, centre, china, culture, jiangsu, suzhou, peoplesrepublicofchina',\n",
       " 259: 'It has tags of arts, centre, china, culture, jiangsu, suzhou, peoplesrepublicofchina',\n",
       " 268: 'It has tags of china, jiangsu, jinji, lake, suzhou, peoplesrepublicofchina',\n",
       " 277: 'It has tags of china, jiangsu, jinji, lake, suzhou, peoplesrepublicofchina',\n",
       " 286: 'It has tags of center, central, china, jiangsu, plaza, square, suzhou, peoplesrepublicofchina',\n",
       " 290: 'It has tags of arts, centre, china, culture, jiangsu, suzhou, peoplesrepublicofchina',\n",
       " 319: 'It has title of Suzhou',\n",
       " 320: 'It has tags of jiangsu, suzhou, jinjihu, peoplesrepublicofchina',\n",
       " 321: 'It has title of Lotus@Jinji Lake Suzhou',\n",
       " 322: 'It has title of Lotus@Jinji Lake Suzhou',\n",
       " 323: 'It has tags of lotus, suzhou, jiangsu, peoplesrepublicofchina',\n",
       " 324: 'It has title of Lotus@Jinji Lake Suzhou',\n",
       " 325: 'It has tags of suzhou, jiangsu, peoplesrepublicofchina, lotus',\n",
       " 342: 'It has title of Suzhou',\n",
       " 343: 'It has tags of suzhou, jiangsu, china',\n",
       " 386: 'It has title of Panmen Gate Park',\n",
       " 387: 'It has title of Garden',\n",
       " 389: 'It has title of Shanghai',\n",
       " 391: 'It has tags of suzhouindustrialparkdongfangzhimen',\n",
       " 523: 'It has title of China',\n",
       " 524: 'Suzhou , China',\n",
       " 525: 'It has title of Beautiful China',\n",
       " 526: 'It has tags of china, suzhou, night',\n",
       " 527: 'It has title of Old room',\n",
       " 529: 'It has tags of china, shanghai, sony',\n",
       " 531: 'It has tags of china, shanghai, sony',\n",
       " 533: 'It has tags of china, shanghai, sony',\n",
       " 535: 'It has tags of china, shanghai, sony',\n",
       " 537: 'It has tags of china, shanghai, sony',\n",
       " 539: 'It has tags of china, shanghai, sony',\n",
       " 540: 'It has title of Shanghai 2018',\n",
       " 541: 'It has tags of red',\n",
       " 544: 'It has title of hidden gold',\n",
       " 545: 'It has title of Autumn',\n",
       " 546: 'It has tags of china, suzhou, central, hotel',\n",
       " 547: 'It has tags of china, suzhou, central, hotel',\n",
       " 548: 'It has tags of china, suzhou',\n",
       " 549: 'It has tags of china, suzhou, central, hotel',\n",
       " 550: 'It has tags of china, suzhou',\n",
       " 551: 'It has tags of china, suzhou',\n",
       " 552: 'It has tags of china, suzhou',\n",
       " 553: 'It has tags of china, suzhou',\n",
       " 554: 'It has tags of china, suzhou',\n",
       " 555: 'It has tags of china, suzhou',\n",
       " 556: 'It has tags of china, suzhou',\n",
       " 557: 'It has tags of china, suzhou',\n",
       " 558: 'It has tags of china, suzhou',\n",
       " 559: 'It has tags of china, suzhou',\n",
       " 560: 'It has tags of china, suzhou',\n",
       " 561: 'It has tags of china, suzhou',\n",
       " 562: 'It has tags of china, suzhou',\n",
       " 563: 'It has tags of china, suzhou',\n",
       " 564: 'It has tags of china, suzhou',\n",
       " 565: 'It has tags of china, suzhou',\n",
       " 566: 'It has tags of china, suzhou',\n",
       " 567: 'It has tags of china, suzhou',\n",
       " 568: 'It has tags of china, suzhou',\n",
       " 569: 'It has tags of china, suzhou',\n",
       " 570: 'It has tags of china, suzhou',\n",
       " 571: 'It has tags of china, suzhou',\n",
       " 572: 'It has tags of china, suzhou',\n",
       " 573: 'It has tags of china, suzhou',\n",
       " 574: 'It has tags of china, suzhou',\n",
       " 575: 'It has tags of china, suzhou',\n",
       " 576: 'It has tags of china, suzhou',\n",
       " 577: 'It has tags of china, suzhou',\n",
       " 578: 'It has tags of china, suzhou',\n",
       " 579: 'It has tags of china, suzhou',\n",
       " 580: 'It has tags of china, suzhou',\n",
       " 581: 'It has tags of china, suzhou',\n",
       " 582: 'It has tags of china, suzhou',\n",
       " 583: 'It has tags of china, suzhou',\n",
       " 584: 'It has tags of china, suzhou',\n",
       " 585: 'It has tags of china, suzhou',\n",
       " 586: 'It has tags of china, suzhou',\n",
       " 587: 'It has tags of china, suzhou',\n",
       " 588: 'It has tags of china, suzhou',\n",
       " 589: 'It has tags of china, suzhou',\n",
       " 590: 'It has tags of china, suzhou',\n",
       " 591: 'It has tags of china, suzhou',\n",
       " 592: 'It has tags of china, suzhou',\n",
       " 593: 'It has tags of china, suzhou',\n",
       " 594: 'It has tags of china, suzhou',\n",
       " 595: 'It has tags of china, suzhou',\n",
       " 596: 'It has tags of china, suzhou',\n",
       " 597: 'It has tags of china, suzhou',\n",
       " 598: 'It has tags of china, suzhou',\n",
       " 599: 'It has tags of china, suzhou',\n",
       " 600: 'It has tags of china, suzhou',\n",
       " 601: 'It has tags of china, suzhou',\n",
       " 602: 'It has tags of china, suzhou',\n",
       " 603: 'It has tags of china, suzhou',\n",
       " 604: 'It has tags of china, suzhou',\n",
       " 605: 'It has tags of china, suzhou',\n",
       " 606: 'It has tags of china, suzhou',\n",
       " 607: 'It has tags of china, suzhou',\n",
       " 608: 'It has tags of china, suzhou',\n",
       " 609: 'It has tags of china, suzhou',\n",
       " 610: 'It has tags of china, suzhou',\n",
       " 611: 'It has tags of china, suzhou',\n",
       " 612: 'It has tags of china, suzhou',\n",
       " 613: 'It has tags of china, suzhou',\n",
       " 614: 'It has tags of china, suzhou',\n",
       " 615: 'It has tags of china, suzhou',\n",
       " 616: 'It has tags of china, suzhou',\n",
       " 617: 'It has tags of china, suzhou',\n",
       " 618: 'It has tags of china, suzhou',\n",
       " 619: 'It has tags of china, suzhou',\n",
       " 620: 'It has tags of china, suzhou',\n",
       " 621: 'It has tags of china, suzhou',\n",
       " 622: 'It has tags of china, suzhou',\n",
       " 623: 'It has tags of china, suzhou',\n",
       " 624: 'It has tags of china, suzhou',\n",
       " 625: 'It has tags of china, suzhou',\n",
       " 626: 'It has tags of china, suzhou',\n",
       " 627: 'It has tags of china, suzhou',\n",
       " 628: 'It has tags of china, suzhou',\n",
       " 629: 'It has title of Suzhou Silk Mill',\n",
       " 630: 'It has title of Suzhou Silk Mill',\n",
       " 631: 'It has title of Suzhou Silk Mill',\n",
       " 632: 'It has title of Suzhou',\n",
       " 633: 'It has tags of suzhou, jeans, china',\n",
       " 634: 'It has tags of suzhou, jeans, china',\n",
       " 635: 'It has tags of suzhou, jeans, china',\n",
       " 636: 'It has tags of suzhou, jeans, china',\n",
       " 637: 'It has tags of suzhou, jeans, china',\n",
       " 638: 'It has tags of suzhou, jeans, china',\n",
       " 639: 'It has tags of suzhou, jeans, china',\n",
       " 640: 'It has tags of suzhou, jeans, china',\n",
       " 641: 'It has tags of suzhou, jeans, china',\n",
       " 642: 'It has tags of suzhou, jeans, china',\n",
       " 643: 'It has tags of suzhou, jeans, china',\n",
       " 644: 'It has tags of suzhou, jeans, china',\n",
       " 645: 'It has tags of suzhou, jeans, china',\n",
       " 684: 'It has title of Shanghai18_124',\n",
       " 685: 'It has title of Shanghai18_108',\n",
       " 686: 'It has title of Shanghai18_106',\n",
       " 687: 'It has title of Shanghai18_103',\n",
       " 704: 'It has tags of suzhou, china, humble, administrator, garden',\n",
       " 706: 'It has tags of suzhou, china, humble, administrator, garden',\n",
       " 708: 'It has tags of suzhou, china, humble, administrator, garden',\n",
       " 710: 'It has tags of suzhou, china, humble, administrator, garden',\n",
       " 712: 'It has tags of suzhou, china',\n",
       " 714: 'It has tags of suzhou, china',\n",
       " 716: 'It has tags of suzhou, china',\n",
       " 718: 'It has tags of suzhou, china',\n",
       " 720: 'It has tags of suzhou, china',\n",
       " 722: 'It has tags of suzhou, china',\n",
       " 724: 'It has tags of suzhou, china',\n",
       " 726: 'It has tags of suzhou, china',\n",
       " 728: 'It has tags of suzhou, china',\n",
       " 730: 'It has tags of suzhou, china',\n",
       " 732: 'It has tags of suzhou, china',\n",
       " 734: 'It has tags of suzhou, china',\n",
       " 736: 'It has tags of suzhou, china',\n",
       " 738: 'It has tags of suzhou, china',\n",
       " 741: 'It has tags of suzhou, china',\n",
       " 754: 'It has title of image',\n",
       " 764: 'It has title of Sūzhōu',\n",
       " 773: 'It has tags of china, places',\n",
       " 775: 'It has tags of china, places',\n",
       " 777: 'It has tags of china, places',\n",
       " 890: 'Lingering garden, Suzhou, China',\n",
       " 930: 'It has title of Autumn nostalgia',\n",
       " 931: 'It has tags of autumn',\n",
       " 932: 'It has title of mmexport1512464899652',\n",
       " 933: 'It has title of mmexport1512464923705',\n",
       " 934: 'It has title of Silk Workers',\n",
       " 940: 'It has title of Vase details, Humble Administrator Garden - Suzhou, Garden',\n",
       " 941: 'It has tags of china, dragon, vase, suzhou, garden',\n",
       " 951: 'It has tags of hangzhou, china',\n",
       " 953: 'It has tags of hangzhou, china',\n",
       " 955: 'It has tags of china, hangzhou',\n",
       " 957: 'It has tags of hangzhou, china',\n",
       " 959: 'It has tags of hangzhou, china',\n",
       " 961: 'It has tags of hangzhou, china',\n",
       " 963: 'It has tags of hangzhou, china',\n",
       " 965: 'It has tags of hangzhou, china',\n",
       " 967: 'It has tags of hangzhou, china',\n",
       " 969: 'It has tags of hangzhou, china',\n",
       " 971: 'It has tags of hangzhou, china',\n",
       " 973: 'It has tags of hangzhou, china',\n",
       " 975: 'It has tags of hangzhou, china',\n",
       " 977: 'It has tags of china, hangzhou',\n",
       " 1025: 'Suzhou Shu China',\n",
       " 1026: 'It has title of Barber',\n",
       " 1027: 'It has title of Suzhou Shu',\n",
       " 1028: 'It has title of Walkway',\n",
       " 1099: 'It has tags of center, central, china, jiangsu, plaza, square, suzhou, peoplesrepublicofchina',\n",
       " 1101: 'It has tags of center, central, china, jiangsu, plaza, square, suzhou, peoplesrepublicofchina',\n",
       " 1104: 'It has tags of center, central, china, jiangsu, plaza, square, suzhou, peoplesrepublicofchina',\n",
       " 1106: 'It has tags of center, central, china, jiangsu, plaza, square, suzhou, peoplesrepublicofchina',\n",
       " 1137: 'It has tags of china, jiangsu, pingjiang, suzhou, peoplesrepublicofchina',\n",
       " 1145: 'It has tags of china, jiangsu, jinji, lake, suzhou, peoplesrepublicofchina',\n",
       " 1147: 'It has tags of china, jiangsu, jinji, lake, suzhou, peoplesrepublicofchina',\n",
       " 1154: 'It has tags of center, central, china, jiangsu, plaza, square, suzhou, peoplesrepublicofchina',\n",
       " 1156: 'It has tags of center, central, china, jiangsu, plaza, square, suzhou, peoplesrepublicofchina',\n",
       " 1159: 'It has tags of arts, centre, china, culture, jiangsu, suzhou, peoplesrepublicofchina',\n",
       " 1162: 'It has tags of arts, centre, china, culture, jiangsu, suzhou, peoplesrepublicofchina',\n",
       " 1172: 'It has tags of china, jiangsu, jinji, lake, suzhou, peoplesrepublicofchina',\n",
       " 1173: 'Battle of Peitsang 05',\n",
       " 1174: 'It has tags of news, china, war',\n",
       " 1196: '2018 Nov Nanjing Jiangsu China',\n",
       " 1198: 'It has tags of china, jiangsu',\n",
       " 1199: '2018 Nov Nanjing Jiangsu China',\n",
       " 1201: 'It has tags of china, jiangsu',\n",
       " 1245: 'It has tags of china, shanghai, sony',\n",
       " 1247: 'It has tags of china, shanghai, sony',\n",
       " 1249: 'It has tags of china, shanghai, sony',\n",
       " 1250: 'It has title of Freshmeat',\n",
       " 1251: 'It has title of Fireball',\n",
       " 1252: 'It has tags of chinasuzhou',\n",
       " 1253: 'It has tags of red',\n",
       " 1254: 'It has tags of china, suzhou',\n",
       " 1255: 'It has tags of china, suzhou',\n",
       " 1256: 'It has tags of china, suzhou',\n",
       " 1257: 'It has tags of china, suzhou',\n",
       " 1258: 'It has tags of china, suzhou',\n",
       " 1259: 'It has tags of china, suzhou',\n",
       " 1260: 'It has tags of china, suzhou',\n",
       " 1261: 'It has tags of china, suzhou',\n",
       " 1262: 'It has tags of china, suzhou',\n",
       " 1263: 'It has tags of china, suzhou',\n",
       " 1264: 'It has tags of china, suzhou',\n",
       " 1265: 'It has tags of china, suzhou',\n",
       " 1266: 'It has tags of china, suzhou',\n",
       " 1267: 'It has tags of china, suzhou',\n",
       " 1268: 'It has tags of china, suzhou',\n",
       " 1269: 'It has tags of china, suzhou',\n",
       " 1270: 'It has tags of china, suzhou',\n",
       " 1271: 'It has tags of china, suzhou',\n",
       " 1272: 'It has tags of china, suzhou',\n",
       " 1273: 'It has tags of china, suzhou',\n",
       " 1277: 'It has tags of suzhou, china, masterofnetsgarden',\n",
       " 1282: 'It has tags of suzhou, china, masterofnetsgarden',\n",
       " 1284: 'It has tags of suzhou, china, masterofnetsgarden',\n",
       " 1286: 'It has tags of suzhou, china, masterofnetsgarden',\n",
       " 1289: 'It has tags of suzhou, china, masterofnetsgarden',\n",
       " 1292: 'It has tags of suzhou, china, masterofnetsgarden',\n",
       " 1297: 'It has tags of suzhou, china, masterofnetsgarden',\n",
       " 1299: 'It has tags of suzhou, china, masterofnetsgarden',\n",
       " 1304: 'It has tags of suzhou, china, masterofnetsgarden',\n",
       " 1306: 'It has tags of suzhou, china, humble, administrator, garden',\n",
       " 1308: 'It has tags of suzhou, china, humble, administrator, garden',\n",
       " 1310: 'It has tags of suzhou, china, humble, administrator, garden',\n",
       " 1312: 'It has tags of suzhou, china, humble, administrator, garden',\n",
       " 1314: 'It has tags of suzhou, china',\n",
       " 1322: 'It has tags of china, places',\n",
       " 1324: 'It has tags of china, places',\n",
       " 1327: 'It has tags of suzhou, china, masterofnetsgarden',\n",
       " 1329: 'It has tags of suzhou, china, masterofnetsgarden',\n",
       " 1331: 'It has tags of suzhou, china, masterofnetsgarden',\n",
       " 1333: 'It has tags of suzhou, china, masterofnetsgarden',\n",
       " 1335: 'It has tags of suzhou, china, humble, administrator, garden',\n",
       " 1377: 'It has tags of suzhou, jiangsu, jiangnan, china, xt, fuji, fujifilm, fujifilmxf',\n",
       " 1378: 'It has tags of suzhou, jiangsu, jiangnan, china, xt, fuji, fujifilm, fujifilmxf',\n",
       " 1379: 'It has tags of suzhou, jiangsu, jiangnan, china, xt, fuji, fujifilm, fuji',\n",
       " 1380: 'It has tags of suzhou, jiangsu, jiangnan, china, xt, fuji, fujifilm, fujifilmxf',\n",
       " 1383: 'It has tags of hangzhou, china',\n",
       " 1385: 'It has tags of hangzhou, china',\n",
       " 1403: 'It has title of Suzhou day1',\n",
       " 1404: 'It has title of barber, Suzhou',\n",
       " 1405: 'It has tags of china, jiangsu, suzhou, zh, barber, cn',\n",
       " 1406: 'It has title of barber, Suzhou',\n",
       " 1407: 'It has title of Changmen gate, Suzhou',\n",
       " 1408: 'It has tags of china, jiangsu, suzhou, zh, gate, cn',\n",
       " 1409: 'It has title of pagoda, Suzhou',\n",
       " 1410: 'It has tags of china, jiangsu, suzhou, zh, pagoda, cn',\n",
       " 1411: 'It has title of Suzhou',\n",
       " 1412: 'It has tags of china, jiangsu, suzhou, zh, skyline, cn',\n",
       " 1442: 'It has title of Xuanmiao temple, Suzhou',\n",
       " 1443: 'It has tags of china, jiangsu, suzhou, zh, temple, cn',\n",
       " 1444: 'Ieoh Ming Pei, 2006',\n",
       " 1445: 'It has title of Suzhou museum',\n",
       " 1446: 'Ieoh Ming Pei, 2006',\n",
       " 1447: 'It has title of Suzhou museum',\n",
       " 1448: 'It has title of Zhuozheng yuan, Suzhou',\n",
       " 1449: 'It has tags of china, jiangsu, suzhou, zh, cn',\n",
       " 1450: 'Ieoh Ming Pei, 2006',\n",
       " 1451: 'It has title of Suzhou museum',\n",
       " 1452: 'It has title of Zhuozheng yuan, Suzhou',\n",
       " 1453: 'It has tags of china, jiangsu, suzhou, zh, garden, tree, cn',\n",
       " 1454: 'It has title of Zhuozheng yuan, Suzhou',\n",
       " 1455: 'It has title of Zhuozheng yuan, Suzhou',\n",
       " 1456: 'It has title of Zhuozheng yuan, Suzhou',\n",
       " 1457: 'It has tags of china, jiangsu, suzhou, zh, garden, pond, cn',\n",
       " 1458: 'It has title of Zhuozheng yuan, Suzhou',\n",
       " 1461: 'It has tags of china',\n",
       " 1463: 'It has tags of suzhoushi, jiangsusheng, china, cn',\n",
       " 1465: 'It has tags of suzhoushi, jiangsusheng, china, cn',\n",
       " 1467: 'It has tags of suzhoushi, jiangsusheng, china, cn',\n",
       " 1469: 'It has tags of suzhoushi, jiangsusheng, china, cn',\n",
       " 1471: 'It has tags of suzhoushi, jiangsusheng, china, cn',\n",
       " 1473: 'It has tags of suzhoushi, jiangsusheng, china, cn',\n",
       " 1475: 'It has tags of suzhoushi, jiangsusheng, china, cn',\n",
       " 1477: 'It has tags of suzhoushi, jiangsusheng, china, cn',\n",
       " 1479: 'It has tags of suzhoushi, jiangsusheng, china, cn',\n",
       " 1481: 'It has tags of suzhoushi, jiangsusheng, china, cn',\n",
       " 1483: 'It has tags of suzhoushi, jiangsusheng, china, cn',\n",
       " 1485: 'It has tags of suzhoushi, jiangsusheng, china, cn',\n",
       " 1487: 'It has tags of suzhoushi, jiangsusheng, china, cn',\n",
       " 1489: 'It has tags of suzhoushi, jiangsusheng, china, cn',\n",
       " 1510: '2018 Nov Nanjing Jiangsu China',\n",
       " 1512: 'It has tags of china, jiangsu',\n",
       " 1513: '2018 Nov Nanjing Jiangsu China',\n",
       " 1515: 'It has tags of china, jiangsu',\n",
       " 1550: 'It has tags of china, shanghai, sony',\n",
       " 1551: 'It has tags of china, suzhou',\n",
       " 1554: 'It has tags of suzhou, jeans, china',\n",
       " 1555: 'It has tags of suzhou, jeans, china',\n",
       " 1556: 'It has tags of suzhou, jeans, china',\n",
       " 1557: 'It has tags of suzhou, jeans, china',\n",
       " 1558: 'It has tags of suzhou, jeans, china',\n",
       " 1559: 'It has tags of suzhou, jeans, china',\n",
       " 1560: 'It has tags of suzhou, jeans, china',\n",
       " 1573: 'It has tags of china, places',\n",
       " 1576: 'It has tags of china, places',\n",
       " 1578: 'It has tags of china, places',\n",
       " 1580: 'It has tags of china, places',\n",
       " 1582: 'It has tags of china, places',\n",
       " 1584: 'It has tags of china, places',\n",
       " 1586: 'It has tags of china, places',\n",
       " 1588: 'It has tags of china, places',\n",
       " 1590: 'It has tags of china, places',\n",
       " 1592: 'It has tags of china, places',\n",
       " 1594: 'It has tags of china, places',\n",
       " 1596: 'It has tags of china, places',\n",
       " 1598: 'It has tags of china, places',\n",
       " 1600: 'It has tags of china, places',\n",
       " 1602: 'It has tags of china, places',\n",
       " 1604: 'It has tags of china, places',\n",
       " 1606: 'It has tags of china, places',\n",
       " 1608: 'It has tags of china, places',\n",
       " 1610: 'It has tags of china, places',\n",
       " 1612: 'It has tags of china, places',\n",
       " 1614: 'It has tags of china, places',\n",
       " 1616: 'It has tags of china, places',\n",
       " 1618: 'It has tags of china, places',\n",
       " 1620: 'It has tags of china, places',\n",
       " 1622: 'It has tags of china, places',\n",
       " 1624: 'It has tags of china, places',\n",
       " 1626: 'It has tags of china, places',\n",
       " 1628: 'It has tags of china, places',\n",
       " 1630: 'It has tags of china, places',\n",
       " 1632: 'It has tags of china, places',\n",
       " 1634: 'It has tags of china, places',\n",
       " 1636: 'It has tags of china, places',\n",
       " 1638: 'It has tags of china, places',\n",
       " 1640: 'It has tags of china, places',\n",
       " 1642: 'It has tags of china, places',\n",
       " 1644: 'It has tags of china, places',\n",
       " 1646: 'It has tags of china, places',\n",
       " 1648: 'It has tags of china, places',\n",
       " 1670: 'It has tags of suzhou, china, gardens',\n",
       " 1671: 'It has tags of suzhou, china',\n",
       " 1675: 'It has title of Emptiness',\n",
       " 1676: 'It has title of Suzhou 7',\n",
       " 1677: 'It has title of Suzhou 5',\n",
       " 1678: 'It has title of Suzhou 3',\n",
       " 1679: 'It has title of Suzhou 2',\n",
       " 1680: 'It has title of Suzhou 1',\n",
       " 1682: 'It has tags of center, central, china, jiangsu, plaza, square, suzhou, peoplesrepublicofchina',\n",
       " 1685: 'It has tags of center, central, china, jiangsu, plaza, square, suzhou, peoplesrepublicofchina',\n",
       " 1712: 'It has tags of china, places',\n",
       " 1714: 'It has tags of china, places',\n",
       " 1716: 'It has tags of china, places',\n",
       " 1718: 'It has tags of china, places',\n",
       " 1720: 'It has tags of china, places',\n",
       " 1722: 'It has tags of china, places',\n",
       " 1724: 'It has tags of china, places',\n",
       " 1726: 'It has tags of china, places',\n",
       " 1728: 'It has tags of china, places',\n",
       " 1730: 'It has tags of china, places',\n",
       " 1732: 'It has tags of china, places',\n",
       " 1742: 'It has tags of china, places',\n",
       " 1744: 'It has tags of china, places',\n",
       " 1748: 'It has tags of china, places',\n",
       " 1761: 'It has tags of suzhoushi, jiangsusheng, china, cn',\n",
       " 1763: 'It has tags of suzhoushi, jiangsusheng, china, cn',\n",
       " 1764: 'Canglang Pavilion 20171120 adam0730',\n",
       " 1773: 'It has tags of chinasuzhou',\n",
       " 1774: 'It has tags of cinasuzhouautumn'}"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filter_dict_other"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4caa49b",
   "metadata": {},
   "source": [
    "#### A Text-only Dataset with separate sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "50407951",
   "metadata": {},
   "outputs": [],
   "source": [
    "filter_df_en = pd.DataFrame()\n",
    "filter_df_en['sentence'] = pd.Series(filter_dict_en)\n",
    "filter_df_en['origin'] = filter_en\n",
    "filter_df_en['lang'] = 'en'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "6c213e00",
   "metadata": {},
   "outputs": [],
   "source": [
    "filter_df_cn = pd.DataFrame()\n",
    "filter_df_cn['sentence'] = pd.Series(filter_dict_cn)\n",
    "filter_df_cn['origin'] = filter_cn\n",
    "filter_df_cn['lang'] = 'it'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "16403969",
   "metadata": {},
   "outputs": [],
   "source": [
    "filter_df_other = pd.DataFrame()\n",
    "filter_df_other['sentence'] = pd.Series(filter_dict_other)\n",
    "filter_df_other['origin'] = filter_other\n",
    "filter_df_other['lang'] = 'other'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "751ba1a7",
   "metadata": {},
   "source": [
    "In total, 984 sentences coming from 754 original posts are saved for further text analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "44499387",
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = pd.concat([filter_df_en,filter_df_cn,filter_df_other]).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "56154b1f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "754"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sentences['origin'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "020b6c2d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>sentence</th>\n",
       "      <th>origin</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>It has title of \"Raining Soochow\"</td>\n",
       "      <td>104</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>It has title of \"Glow In the Dark\"</td>\n",
       "      <td>106</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>and a family (in the background)</td>\n",
       "      <td>131</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>It has title of Goldfishes</td>\n",
       "      <td>131</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>It has title of Tetrahedral shadow and rockery</td>\n",
       "      <td>132</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>979</th>\n",
       "      <td>1761</td>\n",
       "      <td>It has tags of suzhoushi, jiangsusheng, china, cn</td>\n",
       "      <td>4116</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>980</th>\n",
       "      <td>1763</td>\n",
       "      <td>It has tags of suzhoushi, jiangsusheng, china, cn</td>\n",
       "      <td>4117</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>981</th>\n",
       "      <td>1764</td>\n",
       "      <td>Canglang Pavilion 20171120 adam0730</td>\n",
       "      <td>4129</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>982</th>\n",
       "      <td>1773</td>\n",
       "      <td>It has tags of chinasuzhou</td>\n",
       "      <td>4139</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>983</th>\n",
       "      <td>1774</td>\n",
       "      <td>It has tags of cinasuzhouautumn</td>\n",
       "      <td>4140</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>984 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     index                                           sentence  origin   lang\n",
       "0        0                  It has title of \"Raining Soochow\"     104     en\n",
       "1        1                 It has title of \"Glow In the Dark\"     106     en\n",
       "2        5                   and a family (in the background)     131     en\n",
       "3        6                         It has title of Goldfishes     131     en\n",
       "4        7     It has title of Tetrahedral shadow and rockery     132     en\n",
       "..     ...                                                ...     ...    ...\n",
       "979   1761  It has tags of suzhoushi, jiangsusheng, china, cn    4116  other\n",
       "980   1763  It has tags of suzhoushi, jiangsusheng, china, cn    4117  other\n",
       "981   1764                Canglang Pavilion 20171120 adam0730    4129  other\n",
       "982   1773                         It has tags of chinasuzhou    4139  other\n",
       "983   1774                    It has tags of cinasuzhouautumn    4140  other\n",
       "\n",
       "[984 rows x 4 columns]"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "069bc578",
   "metadata": {},
   "source": [
    "#### Finalizing the new Subset with valid textual data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "615565f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "candidates_new = candidates.loc[list(set(filter_en).union(set(filter_cn)).union(set(filter_other)))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "3edfbfbd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>candownload</th>\n",
       "      <th>url_c</th>\n",
       "      <th>url_q</th>\n",
       "      <th>url_n</th>\n",
       "      <th>url_largest</th>\n",
       "      <th>owner</th>\n",
       "      <th>owner_loc</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>...</th>\n",
       "      <th>url</th>\n",
       "      <th>local</th>\n",
       "      <th>tag_list</th>\n",
       "      <th>tag_len</th>\n",
       "      <th>NOT</th>\n",
       "      <th>year</th>\n",
       "      <th>time</th>\n",
       "      <th>week</th>\n",
       "      <th>unique_week</th>\n",
       "      <th>text_data</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>51859675789</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185967578...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185967578...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185967578...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185967578...</td>\n",
       "      <td>57210421@N04</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>灭渡桥</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>https://www.flickr.com/photos/wishingbone/5185...</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>2022</td>\n",
       "      <td>2022-02-03 13:33:40</td>\n",
       "      <td>5</td>\n",
       "      <td>202205</td>\n",
       "      <td>. It has title ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>51858385192</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838519...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838519...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838519...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838519...</td>\n",
       "      <td>57210421@N04</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>下津桥</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>https://www.flickr.com/photos/wishingbone/5185...</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>2022</td>\n",
       "      <td>2022-02-03 14:24:46</td>\n",
       "      <td>5</td>\n",
       "      <td>202205</td>\n",
       "      <td>. It has title ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4111</th>\n",
       "      <td>27910400829</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/4751/27910400829...</td>\n",
       "      <td>https://live.staticflickr.com/4751/27910400829...</td>\n",
       "      <td>https://live.staticflickr.com/4751/27910400829...</td>\n",
       "      <td>https://live.staticflickr.com/4751/27910400829...</td>\n",
       "      <td>127423136@N05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Corridor, Hanshan Temple - Suzhou, China</td>\n",
       "      <td>OLYMPUS DIGITAL CAMERA</td>\n",
       "      <td>...</td>\n",
       "      <td>https://www.flickr.com/photos/127423136@N05/27...</td>\n",
       "      <td>False</td>\n",
       "      <td>[china, buddhism, temple, suzhou, hanshan]</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>2018</td>\n",
       "      <td>2018-01-05 12:08:29</td>\n",
       "      <td>1</td>\n",
       "      <td>201801</td>\n",
       "      <td>OLYMPUS DIGITAL CAMERA. It has title of Corrid...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>51858384372</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838437...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838437...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838437...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838437...</td>\n",
       "      <td>57210421@N04</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>上津桥</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>https://www.flickr.com/photos/wishingbone/5185...</td>\n",
       "      <td>True</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>2022</td>\n",
       "      <td>2022-02-03 14:41:14</td>\n",
       "      <td>5</td>\n",
       "      <td>202205</td>\n",
       "      <td>. It has title ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4116</th>\n",
       "      <td>38919131931</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/4544/38919131931...</td>\n",
       "      <td>https://live.staticflickr.com/4544/38919131931...</td>\n",
       "      <td>https://live.staticflickr.com/4544/38919131931...</td>\n",
       "      <td>https://live.staticflickr.com/4544/38919131931...</td>\n",
       "      <td>95645163@N00</td>\n",
       "      <td>Milan</td>\n",
       "      <td>IMG_6925</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>https://www.flickr.com/photos/opiu/38919131931/</td>\n",
       "      <td>False</td>\n",
       "      <td>[suzhoushi, jiangsusheng, china, cn]</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "      <td>2017</td>\n",
       "      <td>2017-08-22 13:41:19</td>\n",
       "      <td>34</td>\n",
       "      <td>201734</td>\n",
       "      <td>. It has title of IMG_6925. It has tags of suz...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4073</th>\n",
       "      <td>26221985087</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/885/26221985087_...</td>\n",
       "      <td>https://live.staticflickr.com/885/26221985087_...</td>\n",
       "      <td>https://live.staticflickr.com/885/26221985087_...</td>\n",
       "      <td>https://live.staticflickr.com/885/26221985087_...</td>\n",
       "      <td>152311060@N02</td>\n",
       "      <td>Tuxedo Park, USA</td>\n",
       "      <td>Freshmen go to China</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>https://www.flickr.com/photos/tuxedoparkschool...</td>\n",
       "      <td>False</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>2018</td>\n",
       "      <td>2018-03-20 11:59:52</td>\n",
       "      <td>12</td>\n",
       "      <td>201812</td>\n",
       "      <td>. It has title of Freshmen go to China. It has...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4078</th>\n",
       "      <td>41140126964</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/948/41140126964_...</td>\n",
       "      <td>https://live.staticflickr.com/948/41140126964_...</td>\n",
       "      <td>https://live.staticflickr.com/948/41140126964_...</td>\n",
       "      <td>https://live.staticflickr.com/948/41140126964_...</td>\n",
       "      <td>7287512@N02</td>\n",
       "      <td>Kokomo, IN, USA</td>\n",
       "      <td>DSC_20180421_10816</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>https://www.flickr.com/photos/pecasbo/41140126...</td>\n",
       "      <td>False</td>\n",
       "      <td>[china, places]</td>\n",
       "      <td>2</td>\n",
       "      <td>False</td>\n",
       "      <td>2018</td>\n",
       "      <td>2018-04-21 13:57:48</td>\n",
       "      <td>16</td>\n",
       "      <td>201816</td>\n",
       "      <td>. It has title of DSC_20180421_10816. It has t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4079</th>\n",
       "      <td>41814418522</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/957/41814418522_...</td>\n",
       "      <td>https://live.staticflickr.com/957/41814418522_...</td>\n",
       "      <td>https://live.staticflickr.com/957/41814418522_...</td>\n",
       "      <td>https://live.staticflickr.com/957/41814418522_...</td>\n",
       "      <td>7287512@N02</td>\n",
       "      <td>Kokomo, IN, USA</td>\n",
       "      <td>DSC_20180421_10818</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>https://www.flickr.com/photos/pecasbo/41814418...</td>\n",
       "      <td>False</td>\n",
       "      <td>[china, places]</td>\n",
       "      <td>2</td>\n",
       "      <td>False</td>\n",
       "      <td>2018</td>\n",
       "      <td>2018-04-21 13:58:53</td>\n",
       "      <td>16</td>\n",
       "      <td>201816</td>\n",
       "      <td>. It has title of DSC_20180421_10818. It has t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4080</th>\n",
       "      <td>41140126144</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/864/41140126144_...</td>\n",
       "      <td>https://live.staticflickr.com/864/41140126144_...</td>\n",
       "      <td>https://live.staticflickr.com/864/41140126144_...</td>\n",
       "      <td>https://live.staticflickr.com/864/41140126144_...</td>\n",
       "      <td>7287512@N02</td>\n",
       "      <td>Kokomo, IN, USA</td>\n",
       "      <td>DSC_20180421_10825</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>https://www.flickr.com/photos/pecasbo/41140126...</td>\n",
       "      <td>False</td>\n",
       "      <td>[china, family, pedro, people, places]</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>2018</td>\n",
       "      <td>2018-04-21 14:01:20</td>\n",
       "      <td>16</td>\n",
       "      <td>201816</td>\n",
       "      <td>. It has title of DSC_20180421_10825. It has t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4082</th>\n",
       "      <td>41857278401</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/958/41857278401_...</td>\n",
       "      <td>https://live.staticflickr.com/958/41857278401_...</td>\n",
       "      <td>https://live.staticflickr.com/958/41857278401_...</td>\n",
       "      <td>https://live.staticflickr.com/958/41857278401_...</td>\n",
       "      <td>7287512@N02</td>\n",
       "      <td>Kokomo, IN, USA</td>\n",
       "      <td>DSC_20180421_10836</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>https://www.flickr.com/photos/pecasbo/41857278...</td>\n",
       "      <td>False</td>\n",
       "      <td>[china, places]</td>\n",
       "      <td>2</td>\n",
       "      <td>False</td>\n",
       "      <td>2018</td>\n",
       "      <td>2018-04-21 14:05:25</td>\n",
       "      <td>16</td>\n",
       "      <td>201816</td>\n",
       "      <td>. It has title of DSC_20180421_10836. It has t...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>754 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            index  candownload  \\\n",
       "8     51859675789          1.0   \n",
       "13    51858385192          1.0   \n",
       "4111  27910400829          1.0   \n",
       "16    51858384372          1.0   \n",
       "4116  38919131931          1.0   \n",
       "...           ...          ...   \n",
       "4073  26221985087          1.0   \n",
       "4078  41140126964          1.0   \n",
       "4079  41814418522          1.0   \n",
       "4080  41140126144          1.0   \n",
       "4082  41857278401          1.0   \n",
       "\n",
       "                                                  url_c  \\\n",
       "8     https://live.staticflickr.com/65535/5185967578...   \n",
       "13    https://live.staticflickr.com/65535/5185838519...   \n",
       "4111  https://live.staticflickr.com/4751/27910400829...   \n",
       "16    https://live.staticflickr.com/65535/5185838437...   \n",
       "4116  https://live.staticflickr.com/4544/38919131931...   \n",
       "...                                                 ...   \n",
       "4073  https://live.staticflickr.com/885/26221985087_...   \n",
       "4078  https://live.staticflickr.com/948/41140126964_...   \n",
       "4079  https://live.staticflickr.com/957/41814418522_...   \n",
       "4080  https://live.staticflickr.com/864/41140126144_...   \n",
       "4082  https://live.staticflickr.com/958/41857278401_...   \n",
       "\n",
       "                                                  url_q  \\\n",
       "8     https://live.staticflickr.com/65535/5185967578...   \n",
       "13    https://live.staticflickr.com/65535/5185838519...   \n",
       "4111  https://live.staticflickr.com/4751/27910400829...   \n",
       "16    https://live.staticflickr.com/65535/5185838437...   \n",
       "4116  https://live.staticflickr.com/4544/38919131931...   \n",
       "...                                                 ...   \n",
       "4073  https://live.staticflickr.com/885/26221985087_...   \n",
       "4078  https://live.staticflickr.com/948/41140126964_...   \n",
       "4079  https://live.staticflickr.com/957/41814418522_...   \n",
       "4080  https://live.staticflickr.com/864/41140126144_...   \n",
       "4082  https://live.staticflickr.com/958/41857278401_...   \n",
       "\n",
       "                                                  url_n  \\\n",
       "8     https://live.staticflickr.com/65535/5185967578...   \n",
       "13    https://live.staticflickr.com/65535/5185838519...   \n",
       "4111  https://live.staticflickr.com/4751/27910400829...   \n",
       "16    https://live.staticflickr.com/65535/5185838437...   \n",
       "4116  https://live.staticflickr.com/4544/38919131931...   \n",
       "...                                                 ...   \n",
       "4073  https://live.staticflickr.com/885/26221985087_...   \n",
       "4078  https://live.staticflickr.com/948/41140126964_...   \n",
       "4079  https://live.staticflickr.com/957/41814418522_...   \n",
       "4080  https://live.staticflickr.com/864/41140126144_...   \n",
       "4082  https://live.staticflickr.com/958/41857278401_...   \n",
       "\n",
       "                                            url_largest          owner  \\\n",
       "8     https://live.staticflickr.com/65535/5185967578...   57210421@N04   \n",
       "13    https://live.staticflickr.com/65535/5185838519...   57210421@N04   \n",
       "4111  https://live.staticflickr.com/4751/27910400829...  127423136@N05   \n",
       "16    https://live.staticflickr.com/65535/5185838437...   57210421@N04   \n",
       "4116  https://live.staticflickr.com/4544/38919131931...   95645163@N00   \n",
       "...                                                 ...            ...   \n",
       "4073  https://live.staticflickr.com/885/26221985087_...  152311060@N02   \n",
       "4078  https://live.staticflickr.com/948/41140126964_...    7287512@N02   \n",
       "4079  https://live.staticflickr.com/957/41814418522_...    7287512@N02   \n",
       "4080  https://live.staticflickr.com/864/41140126144_...    7287512@N02   \n",
       "4082  https://live.staticflickr.com/958/41857278401_...    7287512@N02   \n",
       "\n",
       "             owner_loc                                     title  \\\n",
       "8       Beijing, China                                       灭渡桥   \n",
       "13      Beijing, China                                       下津桥   \n",
       "4111               NaN  Corridor, Hanshan Temple - Suzhou, China   \n",
       "16      Beijing, China                                       上津桥   \n",
       "4116             Milan                                  IMG_6925   \n",
       "...                ...                                       ...   \n",
       "4073  Tuxedo Park, USA                      Freshmen go to China   \n",
       "4078   Kokomo, IN, USA                        DSC_20180421_10816   \n",
       "4079   Kokomo, IN, USA                        DSC_20180421_10818   \n",
       "4080   Kokomo, IN, USA                        DSC_20180421_10825   \n",
       "4082   Kokomo, IN, USA                        DSC_20180421_10836   \n",
       "\n",
       "                          description  ...  \\\n",
       "8                                      ...   \n",
       "13                                     ...   \n",
       "4111           OLYMPUS DIGITAL CAMERA  ...   \n",
       "16                                     ...   \n",
       "4116                                   ...   \n",
       "...                               ...  ...   \n",
       "4073                                   ...   \n",
       "4078                                   ...   \n",
       "4079                                   ...   \n",
       "4080                                   ...   \n",
       "4082                                   ...   \n",
       "\n",
       "                                                    url  local  \\\n",
       "8     https://www.flickr.com/photos/wishingbone/5185...   True   \n",
       "13    https://www.flickr.com/photos/wishingbone/5185...   True   \n",
       "4111  https://www.flickr.com/photos/127423136@N05/27...  False   \n",
       "16    https://www.flickr.com/photos/wishingbone/5185...   True   \n",
       "4116    https://www.flickr.com/photos/opiu/38919131931/  False   \n",
       "...                                                 ...    ...   \n",
       "4073  https://www.flickr.com/photos/tuxedoparkschool...  False   \n",
       "4078  https://www.flickr.com/photos/pecasbo/41140126...  False   \n",
       "4079  https://www.flickr.com/photos/pecasbo/41814418...  False   \n",
       "4080  https://www.flickr.com/photos/pecasbo/41140126...  False   \n",
       "4082  https://www.flickr.com/photos/pecasbo/41857278...  False   \n",
       "\n",
       "                                        tag_list  tag_len    NOT  year  \\\n",
       "8                                             []        0  False  2022   \n",
       "13                                            []        0  False  2022   \n",
       "4111  [china, buddhism, temple, suzhou, hanshan]        5  False  2018   \n",
       "16                                            []        0  False  2022   \n",
       "4116        [suzhoushi, jiangsusheng, china, cn]        4  False  2017   \n",
       "...                                          ...      ...    ...   ...   \n",
       "4073                                          []        0  False  2018   \n",
       "4078                             [china, places]        2  False  2018   \n",
       "4079                             [china, places]        2  False  2018   \n",
       "4080      [china, family, pedro, people, places]        5  False  2018   \n",
       "4082                             [china, places]        2  False  2018   \n",
       "\n",
       "                    time  week unique_week  \\\n",
       "8    2022-02-03 13:33:40     5      202205   \n",
       "13   2022-02-03 14:24:46     5      202205   \n",
       "4111 2018-01-05 12:08:29     1      201801   \n",
       "16   2022-02-03 14:41:14     5      202205   \n",
       "4116 2017-08-22 13:41:19    34      201734   \n",
       "...                  ...   ...         ...   \n",
       "4073 2018-03-20 11:59:52    12      201812   \n",
       "4078 2018-04-21 13:57:48    16      201816   \n",
       "4079 2018-04-21 13:58:53    16      201816   \n",
       "4080 2018-04-21 14:01:20    16      201816   \n",
       "4082 2018-04-21 14:05:25    16      201816   \n",
       "\n",
       "                                              text_data  \n",
       "8                                    . It has title ...  \n",
       "13                                   . It has title ...  \n",
       "4111  OLYMPUS DIGITAL CAMERA. It has title of Corrid...  \n",
       "16                                   . It has title ...  \n",
       "4116  . It has title of IMG_6925. It has tags of suz...  \n",
       "...                                                 ...  \n",
       "4073  . It has title of Freshmen go to China. It has...  \n",
       "4078  . It has title of DSC_20180421_10816. It has t...  \n",
       "4079  . It has title of DSC_20180421_10818. It has t...  \n",
       "4080  . It has title of DSC_20180421_10825. It has t...  \n",
       "4082  . It has title of DSC_20180421_10836. It has t...  \n",
       "\n",
       "[754 rows x 28 columns]"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "candidates_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "fa6bbe2b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zzbn1\\AppData\\Local\\Temp\\ipykernel_26468\\624303664.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  candidates_new['Chinese'].loc[list(set(filter_cn))] = 1\n",
      "C:\\Users\\zzbn1\\AppData\\Local\\Temp\\ipykernel_26468\\624303664.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  candidates_new['English'].loc[list(set(filter_en))] = 1\n",
      "C:\\Users\\zzbn1\\AppData\\Local\\Temp\\ipykernel_26468\\624303664.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  candidates_new['Other_Lang'].loc[list(set(filter_other))] = 1\n"
     ]
    }
   ],
   "source": [
    "candidates_new['Chinese'] = 0\n",
    "candidates_new['English'] = 0\n",
    "candidates_new['Other_Lang'] = 0\n",
    "candidates_new['Chinese'].loc[list(set(filter_cn))] = 1\n",
    "candidates_new['English'].loc[list(set(filter_en))] = 1\n",
    "candidates_new['Other_Lang'].loc[list(set(filter_other))] = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c600a160",
   "metadata": {},
   "source": [
    "#### Some Statistical Facts of the Subset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb2950ce",
   "metadata": {},
   "source": [
    "Number of unique timestamps (117->66) and owners (95->63) both drop a little bit in the subset compared to the original complete dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "85794a29",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAO2klEQVR4nO3df6zd9V3H8efbIspadwdjXrGgZSnBNVQFrgJTs9ttugJ2qFkiDU4wbM3+IPFHFy3RP/CPxW2RxTDxxw1gjSINwWVA6cL8QWOidYHqpLCujG0obRhlP7xLCXFW3/5xvriT5p7ybe/39vvt+z4fyUnP+Z7v/ZzXPbff1z33c77n+43MRJJUy3f0HUCS1D3LXZIKstwlqSDLXZIKstwlqaAz+g4AcO655+aaNWv6jvH/Xn75ZVauXNl3jImGng+Gn3Ho+WD4GYeeD+pn3Lt371cz800L3pmZvV8uv/zyHJLHHnus7wjHNfR8mcPPOPR8mcPPOPR8mfUzAk/khF51WkaSCrLcJakgy12SCrLcJamgXss9IjZFxNz8/HyfMSSpnF7LPTMfzswtU1NTfcaQpHKclpGkgix3SSpoEJ9QlTQca7Y90mq9reuPclPLddt67sPXdjrecuYrd0kqyHKXpIIsd0kqyHKXpIIsd0kqyE+oSlJBfkJVkgpyWkaSCrLcJakgy12SCrLcJakgy12SCvLAYZIGo+1By9pqe3Czigcs85W7JBVkuUtSQZa7JBVkuUtSQZa7JBXkgcMkqSAPHCZJBTktI0kFWe6SVJDlLkkFWe6SVJDlLkkFWe6SVJDlLkkFWe6SVJDlLkkFWe6SVJDlLkkFWe6SVJDlLkkFWe6SVJDlLkkFWe6SVJDlLkkFWe6SVNAZSzFoRPwccC3weuDuzPz0UjyOJGlhrV+5R8Q9EXE4Ip46ZvnGiDgQEc9GxDaAzPxkZr4f+ADwi91GliS9lhOZltkObBxfEBErgDuBq4F1wOaIWDe2yu8090uSTqHW5Z6Z/wB8/ZjFPw48m5lfysxvATuA62LkI8CnMvNfuosrSWojMrP9yhFrgJ2ZeUlz+z3Axsx8X3P7vcAVwDPAjcDjwGcz808WGGsLsAVgenr68h07dizuO+nQkSNHWLVqVd8xJhp6Phh+xqHng/4y7js032q96bPgxVeWOMwitc24fvXU0oeZYDE/5w0bNuzNzJmF7luSN1Qz8w7gjtdYZw6YA5iZmcnZ2dmliHJSdu/ezZDyHGvo+WD4GYeeD/rLeNO2R1qtt3X9UW7ftyQV0pm2GZ+7YXbpw0ywVD/nxe4KeQi4YOz2+c0ySVKPFlvujwMXRcSFEXEmcD3wUNsvjohNETE3P9/uz0BJUjsnsivkfcAe4OKIOBgRN2fmUeAW4FFgP3B/Zj7ddszMfDgzt0xN9TffJUkVtZ4wy8zNE5bvAnZ1lkiStGgefkCSCuq13J1zl6Sl0Wu5O+cuSUvDaRlJKshyl6SCLHdJKsg3VCWpIN9QlaSCnJaRpIIsd0kqyHKXpIJ8Q1WSCvINVUkqyGkZSSrIcpekgix3SSpo2Ge3lZaxfYfmW5+sWjqWe8tIUkHuLSNJBTnnLkkFOeeuE7Km5Rzw1vVHO50vfu7D13Y2lrQc+Mpdkgqy3CWpIMtdkgqy3CWpIPdzl6SC3M9dkgpyWkaSCrLcJakgy12SCrLcJakgy12SCrLcJakgy12SCur1qJARsQnYtHbt2j5jSFrm2h7tdCls37hyScb1Q0ySVJDTMpJUkOUuSQVZ7pJUkOUuSQVZ7pJUkOUuSQVZ7pJUkOUuSQVZ7pJUkOUuSQVZ7pJUkOUuSQX1Wu4RsSki5ubn5/uMIUnleFRISSrIaRlJKqjXk3Xo5Ow7NM9NPZ5cQNLw+cpdkgqy3CWpIMtdkgqy3CWpIMtdkgqy3CWpIMtdkgqy3CWpIMtdkgqy3CWpIA8/IB3Hmh4P87B1fW8PrQJ85S5JBVnuklSQ5S5JBVnuklSQ5S5JBXVe7hHx5oi4OyIe6HpsSVI7rco9Iu6JiMMR8dQxyzdGxIGIeDYitgFk5pcy8+alCCtJaqftK/ftwMbxBRGxArgTuBpYB2yOiHWdppMknZTIzHYrRqwBdmbmJc3tq4DbMvNdze1bATLz95rbD2Tme44z3hZgC8D09PTlO3bsWMS30a0jR46watWqvmNMdPjr87z4St8pjm/6LDrNuH71VHeD0f5nvO/QfKePeyK6fg67NvR8cHpkvHBqxUn3zYYNG/Zm5sxC9y3mE6qrgefHbh8EroiINwIfAi6NiFtfLftjZeYcMAcwMzOTs7Ozi4jSrd27dzOkPMf6+L0Pcvu+YX+4eOv6o51mfO6G2c7GgvY/4z5PRN71c9i1oeeD0yPj9o0rl6RvOv+uM/NrwAe6HleS1N5i9pY5BFwwdvv8ZpkkqWeLKffHgYsi4sKIOBO4HnjoRAaIiE0RMTc/39+8piRV1HZXyPuAPcDFEXEwIm7OzKPALcCjwH7g/sx8+kQePDMfzswtU1PdvlkmSctdqzn3zNw8YfkuYFeniSRJi+bhBySpoF7L3Tl3SVoavZa7c+6StDSclpGkgix3SSrIOXdJKsg5d0kqyGkZSSrIcpekgix3SSrIcpekgtxbRpIKcm8ZSSrIaRlJKshyl6SCLHdJKshyl6SC3FtGkgpybxlJKshpGUkqyHKXpIIsd0kqyHKXpIIsd0kqyF0hJakgd4WUpIKclpGkgix3SSrIcpekgix3SSrIcpekgix3SSrIcpekgix3SSrojD4fPCI2AZvWrl3bZwydBtZse6TT8bauP8pNHY8pDYmfUJWkgpyWkaSCLHdJKshyl6SCLHdJKshyl6SCLHdJKshyl6SCLHdJKshyl6SCLHdJKshyl6SCLHdJKqjXco+ITRExNz8/32cMSSrHo0JKUkFOy0hSQZa7JBVkuUtSQZa7JBVkuUtSQZa7JBVkuUtSQZa7JBVkuUtSQZa7JBVkuUtSQZa7JBVkuUtSQZa7JBVkuUtSQZa7JBVkuUtSQZa7JBV0RtcDRsRK4I+AbwG7M/Perh9DknR8rV65R8Q9EXE4Ip46ZvnGiDgQEc9GxLZm8S8AD2Tm+4F3d5xXktRC22mZ7cDG8QURsQK4E7gaWAdsjoh1wPnA881q/9NNTEnSiYjMbLdixBpgZ2Ze0ty+CrgtM9/V3L61WfUg8I3M3BkROzLz+gnjbQG2AExPT1++Y8eOk/oG9h2aP6mvO57ps+DFV157vfWrpzp/7DYOf32+Vb4+tX0O+zL0fDD8jEPPB6dHxgunVrBq1aqT+toNGzbszcyZhe5bzJz7ar79Ch1GpX4FcAfwhxFxLfDwpC/OzDlgDmBmZiZnZ2dPKsRN2x45qa87nq3rj3L7vtd+ap67Ybbzx27j4/c+2Cpfn9o+h30Zej4Yfsah54PTI+P2jSs52f47ns6/68x8GfiVrseVJLW3mF0hDwEXjN0+v1nWWkRsioi5+fnup1YkaTlbTLk/DlwUERdGxJnA9cBDJzJAZj6cmVumpvqZu5akqtruCnkfsAe4OCIORsTNmXkUuAV4FNgP3J+ZTy9dVElSW63m3DNz84Tlu4BdnSaSJC1ar4cfcM5dkpZGr+XunLskLQ0PHCZJBbX+hOqShoh4Cfj3vnOMORf4at8hjmPo+WD4GYeeD4afcej5oH7GH8zMNy10xyDKfWgi4olJH+kdgqHng+FnHHo+GH7GoeeD5Z3RaRlJKshyl6SCLPeFzfUd4DUMPR8MP+PQ88HwMw49HyzjjM65S1JBvnKXpIIsd0kqaFmUe0RcEBGPRcTnIuLpiPjVZvk5EfE3EfGF5t+zm+UREXc054Z9MiIuGxvro80Y+5t1ood8PxQReyLivyLig8eMtdB5bRetq4yTxhlKvrHxVkTEv0bEzi7ydZ0xIt4QEQ9ExOeb/4tXDTDjrzdjPBUR90XEd/eQ74ZmG94XEf8UET8yNtZQtpUFMy56W8nM8hfgPOCy5vr3AM8wOu/rR4FtzfJtwEea69cAnwICuBL4TLP8rcA/Aiuayx5gtod83wv8GPAh4INj46wAvgi8GTgT+DdgXU/P4aSMC44zlHxj4/0G8FeMTi3Z1//DiRmBPwfe11w/E3jDkDIyOlPbl4Gzmtv3Azf1kO+twNnN9av59rY8pG1lUsZFbSud/Kc93S7Ag8BPAweA88aeyAPN9T8FNo+tf6C5/ypgL3AW8DrgCeAtpzrf2Hq3HbNBXQU8Onb7VuDWPp7DSRknjTOkfIxOPPN3wNvpsNw7/DlPMSrOWKpsHWR89TSc5zA6+uxO4Gf6ytcsPxs41Fwf3LZybMZJ47R93GUxLTMuRif6vhT4DDCdmS80d30FmG6uL3R+2NWZuQd4DHihuTyamft7yDfJgrm7zNdBxknjDCnfHwC/Cfxvl7nGLTLjhcBLwJ81U0d3RcTKIWXMzEPA7wP/wWhbmc/MT/ec72ZGf5HDcLeV8YyTxmllWZV7RKwC/hr4tcz85vh9OfrVeNz9QiNiLfAWRq/sVgNvj4ifGkq+U6GrjMcbp898EfGzwOHM3NtVpq4zMnolfBnwx5l5KfAyoz/zB5OxmU++jtEvou8HVkbEL/WVLyI2MCrO3+oqw6nKeLLbyrIp94j4TkZP0L2Z+Ylm8YsRcV5z/3nA4Wb5pPPD/jzwz5l5JDOPMPoN29UbWSeSb5JFn9f2FGScNM5Q8v0E8O6IeA7YwegX+F8OLONB4GBmvvoq7gFGZT+kjO8EvpyZL2XmfwOfYDS3fMrzRcQPA3cB12Xm15rFg9pWJmRc1LayLMo9IgK4G9ifmR8bu+sh4Mbm+o2M5rReXf7LMXIloz8pX2D0J+bbIuKM5kl/G6NTDJ7qfJMs+ry2S53xOOMMIl9m3pqZ52fmGkbP399nZievODvM+BXg+Yi4uFn0DuBzQ8rIaFu5MiJe14z5DnrYViLiBxj9YnlvZj4ztv5gtpVJGRe9rSzFGwhDuwA/yehPoCeBzzaXa4A3Mnrj7AvA3wLnNOsHcCejd9P3ATPN8hWM3mzdz2hj+lhP+b6P0au3bwL/2Vx/fXPfNYzeVf8i8Ns9PocLZpw0zlDyHTPmLN3uLdPlz/lHGb2h/yTwSZq9LQaW8XeBzwNPAX8BfFcP+e4CvjG27hNjYw1lW1kw46Rx2ubw8AOSVNCymJaRpOXGcpekgix3SSrIcpekgix3SSrIcpekgix3SSro/wB9BsQbusg3vgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "candidates_new['year'].hist(log=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "cd9a53b9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "66"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(candidates_new['unique_week'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "08c3f0c7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "63"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(candidates_new['owner'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "165f949d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(candidates_new['neighbourhood'].unique())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1af98d2e",
   "metadata": {},
   "source": [
    "#### Merging the translated texts back to a Revised version of text data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e48e5a9",
   "metadata": {},
   "source": [
    "The data are merged back in the order of English, translated Italian, and translated other languages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "3f46ae66",
   "metadata": {},
   "outputs": [],
   "source": [
    "rev = {}\n",
    "keys_en = list(filter_dict_en.keys())\n",
    "keys_cn = list(filter_dict_cn.keys())\n",
    "keys_other = list(filter_dict_other.keys())\n",
    "for i in range(len(filter_en)):\n",
    "    if not filter_en[i] in rev:\n",
    "        rev[filter_en[i]] = filter_dict_en[keys_en[i]]\n",
    "    else:\n",
    "        rev[filter_en[i]] = '. '.join([rev[filter_en[i]], filter_dict_en[keys_en[i]]])\n",
    "        \n",
    "for i in range(len(filter_cn)):\n",
    "    if not filter_cn[i] in rev:\n",
    "        rev[filter_cn[i]] = filter_dict_cn[keys_cn[i]]\n",
    "    else:\n",
    "        rev[filter_cn[i]] = '. '.join([rev[filter_cn[i]], filter_dict_cn[keys_cn[i]]])\n",
    "        \n",
    "for i in range(len(filter_other)):\n",
    "    if not filter_other[i] in rev:\n",
    "        rev[filter_other[i]] = filter_dict_other[keys_other[i]]\n",
    "    else:\n",
    "        rev[filter_other[i]] = '. '.join([rev[filter_other[i]], filter_dict_other[keys_other[i]]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "00293878",
   "metadata": {},
   "outputs": [],
   "source": [
    "candidates_new['revised_text'] = pd.Series(rev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "id": "60161807",
   "metadata": {},
   "outputs": [],
   "source": [
    "candidates_new['text_len'] = candidates_new['revised_text'].apply(lambda x: len(x.split()))\n",
    "candidates_new['num_sent'] = candidates_new['revised_text'].apply(lambda x: len(x.split('. ')))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a96f1b34",
   "metadata": {},
   "source": [
    "Distribution of words per sentence and sentences per data point"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "id": "27df5683",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD4CAYAAAD4k815AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAMTElEQVR4nO3df2xdd3nH8fezZIWSaG5pkbUl1RzkqChqBAULWm2aHIZESubBH0hrVgkqRUSVgLEp0uQK/oA/0DJp7EeliimiXdk0NWMZYm1T8avD2j9QmgBq0oZCYGFNVJqygSEVElh79sc9ka6c2LmO773n8tz3S7KS8/Xpuc/jr/vR9fd8fRKZiSSpjl9ruwBJUn8Z7JJUjMEuScUY7JJUjMEuScVsbLsAgBtvvDGnpqYuGX/55ZfZtGnT8AtqkT2Ph3Hredz6heH0fPz48R9l5muWj49EsE9NTXHs2LFLxhcWFpidnR1+QS2y5/Ewbj2PW78wnJ4j4geXG3cpRpKKMdglqZhWgz0i5iLi0OLiYptlSFIprQZ7Zj6amfsnJibaLEOSSnEpRpKKMdglqRiDXZKKMdglqZhWf0EpIuaAuenp6au+xtT80f4VtAZnDu5p5XUl6UrcFSNJxbgUI0nFGOySVIzBLknFGOySVIzBLknFGOySVIxPd5SkYtzHLknFuBQjScUY7JJUjMEuScUY7JJUjMEuScUY7JJUjMEuScUY7JJUjMEuScX4SAFJKsZHCkhSMS7FSFIxBrskFWOwS1IxBrskFWOwS1IxBrskFWOwS1IxBrskFWOwS1IxBrskFWOwS1IxBrskFbOxzRePiDlgbnp6us0yrsrU/NGBXPfAziXuvsK1zxzcM5DXllSDT3eUpGJcipGkYgx2SSrGYJekYgx2SSrGYJekYgx2SSrGYJekYgx2SSrGYJekYgx2SSrGYJekYgx2SSrGYJekYgx2SSrGYJekYgx2SSrGYJekYgx2SSrGYJekYgx2SSpmY78vGBHvAvYAvwE8kJlf7PdrSJJW1tM79oh4MCLOR8TJZeO7I+K5iDgdEfMAmfm5zHwfcA/wR/0vWZK0ml6XYh4CdncPRMQG4H7gDmAHsDcidnSd8pHm85KkIYrM7O3EiCngscy8pTm+HfhoZr69Ob63OfVg8/GlzPzyKtfbD+wHmJycfNPhw4cvOefChQts3rx51bpOnFvsqf5fFZPXwos/X/2cnVsmhlPMkPQyz9WMW8/j1i8Mp+ddu3Ydz8yZ5ePrWWPfAjzfdXwWeAvwQeBtwERETGfm31/uP87MQ8AhgJmZmZydnb3knIWFBS433u3u+aNXUfroOrBziU+cWH1aztw1O5xihqSXea5m3Hoet36h3Z77fvM0M+8D7uv3dSVJvVnPdsdzwE1dx1ubMUlSi9YT7E8B2yNiW0RcA9wJPLKWC0TEXEQcWlystU4uSW3qdbvjw8BXgZsj4mxE7MvMJeADwBeAU8BnMvOZtbx4Zj6amfsnJmrdDJSkNvW0xp6Ze1cYfxx4vK8VSZLWxUcKSFIxrQa7a+yS1H+tBrtr7JLUfy7FSFIxBrskFWOwS1Ix3jyVpGK8eSpJxbgUI0nFGOySVIzBLknFePNUkorx5qkkFeNSjCQVY7BLUjEGuyQVY7BLUjEGuyQV43ZHSSrG7Y6SVIxLMZJUjMEuScUY7JJUjMEuScUY7JJUjMEuScW4j12SinEfuyQV41KMJBVjsEtSMQa7JBVjsEtSMQa7JBVjsEtSMQa7JBVjsEtSMf7mqSQV42+eSlIxLsVIUjEGuyQVY7BLUjEGuyQVY7BLUjEGuyQVY7BLUjEGuyQVY7BLUjEGuyQVY7BLUjEGuyQVY7BLUjE+tleSivGxvZJUjEsxklSMwS5JxRjsklSMwS5JxRjsklSMwS5JxRjsklSMwS5JxRjsklSMwS5JxRjsklSMwS5JxRjsklSMwS5JxRjsklSMwS5JxRjsklSMwS5JxRjsklRM34M9Il4bEQ9ExJF+X1uSdGU9BXtEPBgR5yPi5LLx3RHxXEScjoh5gMz8fmbuG0SxkqQr6/Ud+0PA7u6BiNgA3A/cAewA9kbEjr5WJ0las8jM3k6MmAIey8xbmuPbgY9m5tub43sBMvMvmuMjmfnuVa63H9gPMDk5+abDhw9fcs6FCxfYvHnzqnWdOLfYU/2/KiavhRd/vvo5O7dMDKeYZQb1tR7lngell+/tSsatXxhOz7t27TqemTPLxzeu45pbgOe7js8Cb4mIG4CPA7dGxL0Xg365zDwEHAKYmZnJ2dnZS85ZWFjgcuPd7p4/ejW1j6wDO5f4xInVp+XMXbPDKWaZQX2tR7nnQenle7uScesX2u15PcF+WZn5P8A9/b6uJKk369kVcw64qet4azMmSWrRet6xPwVsj4htdAL9TuCP13KBiJgD5qanp9dRxviZKrb8JKm/et3u+DDwVeDmiDgbEfsycwn4APAF4BTwmcx8Zi0vnpmPZub+iYlaN8YkqU09vWPPzL0rjD8OPN7XiiRJ6+IjBSSpmL7vilkL19jVq7buK5w5uKeV123TIL7WB3Yu9bRddhy/3oPQ6jt219glqf9cipGkYgx2SSrGYJekYrx5Kq1iUDdte7mZ6I1EXS1vnkpSMS7FSFIxBrskFWOwS1IxBrskFdNqsEfEXEQcWlys9c/bSVKb3BUjScW4FCNJxRjsklSMwS5JxRjsklSMu2IkqRh3xUhSMS7FSFIxBrskFWOwS1IxBrskFWOwS1IxBrskFeM+dkkqxn3sklSMSzGSVIzBLknFGOySVIzBLknFGOySVIzBLknFGOySVIzBLknFbGzzxSNiDpibnp5uswxpJE3NH227hLExiK/1gZ1L3H2F6545uKfvrwv+5qkkleNSjCQVY7BLUjEGuyQVY7BLUjEGuyQVY7BLUjEGuyQVY7BLUjEGuyQVY7BLUjEGuyQVY7BLUjEGuyQV42N7JY0MH1XcHz62V5KKcSlGkoox2CWpGINdkoox2CWpGINdkoox2CWpGINdkoox2CWpmMjMtmsgIl4CfnCZT90I/GjI5bTNnsfDuPU8bv3CcHr+7cx8zfLBkQj2lUTEscycabuOYbLn8TBuPY9bv9Buzy7FSFIxBrskFTPqwX6o7QJaYM/jYdx6Hrd+ocWeR3qNXZK0dqP+jl2StEYGuyQVM5LBHhG7I+K5iDgdEfNt1zMIEXFTRHwlIp6NiGci4kPN+Ksj4ksR8d3mz+vbrrXfImJDRHwzIh5rjrdFxJPNfP9LRFzTdo39FBHXRcSRiPh2RJyKiNurz3NE/FnzfX0yIh6OiFdWm+eIeDAizkfEya6xy85rdNzX9P50RLxxkLWNXLBHxAbgfuAOYAewNyJ2tFvVQCwBBzJzB3Ab8P6mz3ngiczcDjzRHFfzIeBU1/FfAn+TmdPAj4F9rVQ1OH8HfD4zXwe8nk7vZec5IrYAfwLMZOYtwAbgTurN80PA7mVjK83rHcD25mM/8MlBFjZywQ68GTidmd/PzF8Ah4F3tlxT32XmC5n5jebvP6PzP/sWOr1+ujnt08C7WilwQCJiK7AH+FRzHMBbgSPNKaV6jogJ4PeABwAy8xeZ+ROKzzOdf0/52ojYCLwKeIFi85yZ/wn877Lhleb1ncA/ZsfXgOsi4jcHVdsoBvsW4Pmu47PNWFkRMQXcCjwJTGbmC82nfghMtlXXgPwt8OfA/zXHNwA/ycyl5rjafG8DXgL+oVl++lREbKLwPGfmOeCvgP+mE+iLwHFqz/NFK83rUHNtFIN9rETEZuDfgD/NzJ92fy47e1HL7EeNiD8Azmfm8bZrGaKNwBuBT2bmrcDLLFt2KTjP19N5h7oN+C1gE5cuWZTX5ryOYrCfA27qOt7ajJUTEb9OJ9T/OTM/2wy/ePFHtObP823VNwC/A/xhRJyhs8T2Vjrrz9c1P7JDvfk+C5zNzCeb4yN0gr7yPL8N+K/MfCkzfwl8ls7cV57ni1aa16Hm2igG+1PA9uYO+jV0bro80nJNfdesLT8AnMrMv+761CPAe5u/vxf492HXNiiZeW9mbs3MKTrz+h+ZeRfwFeDdzWnVev4h8HxE3NwM/T7wLIXnmc4SzG0R8arm+/xiz2XnuctK8/oI8J5md8xtwGLXkk3/ZebIfQDvAL4DfA/4cNv1DKjH36XzY9rTwLeaj3fQWXN+Avgu8GXg1W3XOqD+Z4HHmr+/Fvg6cBr4V+AVbdfX517fABxr5vpzwPXV5xn4GPBt4CTwT8Arqs0z8DCdewi/pPOT2b6V5hUIOrv9vgecoLNjaGC1+UgBSSpmFJdiJEnrYLBLUjEGuyQVY7BLUjEGuyQVY7BLUjEGuyQV8/8afl1CzMJHOAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "candidates_new['text_len'].hist(log=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "0738a01f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAANKUlEQVR4nO3df4hdd5nH8c/HxGI3s46wKUNJipMlJRAa2NqLpbjIjK42IU4REUyoQqR0LFhRDCyTZWG7fyybf7osrd1dhqZW2WyHGn+0+YGtooO7ULWZWpnWWKgy0hl2O7rdve6EgsR99o+cskPMtCfnntvvvY/vF4TmnHvvuc/TDp/eec73nuOIEAAgl7eULgAA0D7CHQASItwBICHCHQASItwBIKHNpQuQpK1bt8b4+Hij154/f15btmxpt6BC6GXwZOlDopdB1UsvCwsLv4qIay732ECE+/j4uM6ePdvotfPz85qYmGi3oELoZfBk6UOil0HVSy+2f7HRY0XHMranbM92u92SZQBAOkXDPSJORsT06OhoyTIAIB1OqAJAQoQ7ACTEzB0AEmLmDgAJMZYBgIQIdwBIaCC+xNSLxZWuDs2cLvLeS0f3F3lfAHgjnFAFgIQ4oQoACTFzB4CECHcASIhwB4CECHcASIjVMgCQEKtlACAhxjIAkBDhDgAJEe4AkBDhDgAJEe4AkBBLIQEgIZZCAkBCjGUAICHCHQASItwBICHCHQASItwBICHCHQASYp07ACTEOncASIixDAAkRLgDQEKEOwAkRLgDQEKEOwAkRLgDQEKEOwAkRLgDQEKEOwAkRLgDQEKEOwAkxIXDACAhLhwGAAkxlgGAhAh3AEiIcAeAhAh3AEiIcAeAhAh3AEiIcAeAhAh3AEiIcAeAhAh3AEiIcAeAhAh3AEiIcAeAhAh3AEiIcAeAhAh3AEhocz8OavvDkvZLerukYxHxZD/eBwBwebU/udt+yPaq7ecu2b/X9gu2X7Q9I0kR8Y2IuFPSXZI+1m7JAIA3ciVjmYcl7V2/w/YmSQ9I2idpt6SDtneve8pfVo8DAN5Ejoj6T7bHJZ2KiBuq7Vsk3RMRt1bbR6qnHq3+fCsivr3BsaYlTUvS2NjYTXNzc40aWH2lq5dfbfTSnu3Z1u69X9fW1jQyMtLqMUvJ0kuWPiR6GVS99DI5ObkQEZ3LPdbrzH2bpJfWbS9LulnSZyT9maRR2zsj4p8ufWFEzEqalaROpxMTExONCrj/+GO6d7Evpw7e0NLtE60eb35+Xk3/PQyaLL1k6UOil0HVr176kooRcZ+k+/pxbADAG+t1KeSKpOvWbW+v9tVie8r2bLfb7bEMAMB6vYb705Kut73D9lWSDkh6vO6LI+JkREyPjrY7uwaA33dXshTyEUlPSdple9n2HRFxQdLdkp6QdE7SoxHxfH9KBQDUVXvmHhEHN9h/RtKZ1ioCAPSszDKTiu0pSVM7d+4sWUZj4zOnWz3e4T0XdKjGMZeO7m/1fQHkU/TaMszcAaA/uHAYACREuANAQkXDnXXuANAfzNwBICHGMgCQEOEOAAkR7gCQECdUASAhTqgCQEKMZQAgIcIdABIi3AEgIcIdABJitQwAJMRqGQBIiLEMACREuANAQoQ7ACREuANAQqyWAYCENpd884g4Kelkp9O5s2QdqG985nSt5x3ec0GHaj63jqWj+1s7FvD7gLEMACREuANAQoQ7ACREuANAQoQ7ACREuANAQoQ7ACTEl5gAICEu+QsACTGWAYCECHcASIhwB4CECHcASIhwB4CECHcASIhwB4CECHcASIhwB4CECHcASIhwB4CEuHAYACTEhcMAICHGMgCQEOEOAAkR7gCQEOEOAAkR7gCQEOEOAAkR7gCQEOEOAAkR7gCQEOEOAAkR7gCQEOEOAAkR7gCQEOEOAAkR7gCQEOEOAAm1Hu62/9j2Mdsn2j42AKCeWuFu+yHbq7afu2T/Xtsv2H7R9owkRcTPI+KOfhQLAKin7if3hyXtXb/D9iZJD0jaJ2m3pIO2d7daHQCgEUdEvSfa45JORcQN1fYtku6JiFur7SOSFBF/W22fiIiPvs7xpiVNS9LY2NhNc3NzjRpYfaWrl19t9NKBM3a1avWyZ1u5e84urtS7mXndXuoq1fPa2ppGRkaKvHfb6GUw9dLL5OTkQkR0LvfY5h5q2ibppXXby5Jutv1Hkv5G0o22j7wW9peKiFlJs5LU6XRiYmKiURH3H39M9y720sbgOLznQq1elm6f6H8xGzg0c7rW8+r2Ulepnufn59X0Z3PQ0Mtg6lcvradiRPynpLvaPi4AoL5ewn1F0nXrtrdX+2qzPSVpaufOnT2UAfTP4kq39m8rbVs6ur/I+yKHXpZCPi3pets7bF8l6YCkx6/kABFxMiKmR0fLzZABIKO6SyEfkfSUpF22l23fEREXJN0t6QlJ5yQ9GhHP969UAEBdtcYyEXFwg/1nJJ1ptSIAQM+KXn7A9pTt2W633vI6AEA9RcOdmTsA9AcXDgOAhAh3AEiImTsAJMTMHQASYiwDAAkR7gCQEOEOAAkVvVYuFw4DNjbe8gXLDu+5UOsiaFywLAdOqAJAQoxlACAhwh0AEiLcASAhvqEKAAlxQhUAEmIsAwAJEe4AkBDhDgAJEe4AkBDhDgAJsRQSABJiKSQAJMRYBgASItwBICHCHQASItwBICHCHQASItwBICHCHQAS4gbZAAZG3ZuC173Zd10ZbwrOl5gAICHGMgCQEOEOAAkR7gCQEOEOAAkR7gCQEOEOAAkR7gCQEOEOAAkR7gCQEOEOAAkR7gCQUNFwtz1le7bb7ZYsAwDS4cJhAJAQYxkASIhwB4CECHcASIhwB4CECHcASIhwB4CECHcASIhwB4CECHcASIhwB4CECHcASIhwB4CECHcASIhwB4CECHcASIhwB4CENrd9QNtbJP2DpN9Imo+I422/BwDg9dX65G77Idurtp+7ZP9e2y/YftH2TLX7I5JORMSdkm5ruV4AQA11xzIPS9q7foftTZIekLRP0m5JB23vlrRd0kvV037bTpkAgCvhiKj3RHtc0qmIuKHavkXSPRFxa7V9pHrqsqT/iohTtuci4sAGx5uWNC1JY2NjN83NzTVqYPWVrl5+tdFLB87Y1arVy55t5e45u7hS72bmdXupq1TP/Hy9uUr9fJW0Y3STRkZGGr12cnJyISI6l3usl5n7Nv3/J3TpYqjfLOk+SV+wvV/SyY1eHBGzkmYlqdPpxMTERKMi7j/+mO5dbP3UQRGH91yo1cvS7RP9L2YDh2ZO13pe3V7qKtUzP19vrlI/XyU9vHeLmubf62n9305EnJf0ybaPCwCor5elkCuSrlu3vb3aV5vtKduz3W69X8UAAPX0Eu5PS7re9g7bV0k6IOnxKzlARJyMiOnR0XIzPgDIqO5SyEckPSVpl+1l23dExAVJd0t6QtI5SY9GxPP9KxUAUFetmXtEHNxg/xlJZ1qtCADQs6KXH2DmDgD9UTTcmbkDQH9w4TAASKj2N1T7WoT9S0m/aPjyrZJ+1WI5JdHL4MnSh0Qvg6qXXt4ZEddc7oGBCPde2D670ddvhw29DJ4sfUj0Mqj61QtjGQBIiHAHgIQyhPts6QJaRC+DJ0sfEr0Mqr70MvQzdwDA78rwyR0AcAnCHQASGtpw3+i+rsPI9nW2v2v7J7aft/3Z0jU1Yftttn9o+8dVH39duqZe2d5k+0e2T5WupRe2l2wv2n7W9tnS9TRl+x22T9j+qe1z1R3hho7tXdV/i9f+/Nr251p9j2Gdudt+r6Q1SV9+7dZ/w8r2tZKujYhnbP+hpAVJH46InxQu7YrYtqQtEbFm+62S/k3SZyPi+4VLa8z25yV1JL09Ij5Uup6mbC9J6kTEUH/xx/aXJP1rRDxYXWr8DyLivwuX1ZPqftQrkm6OiKZf5vwdQ/vJPSK+J+mV0nW0ISL+PSKeqf7+P7p4CeVtZau6cnHRWrX51urPcH56kGR7u6T9kh4sXQsk26OS3ivpmCRFxG+GPdgr75f0szaDXRricM+quhH5jZJ+ULiURqoxxrOSViV9KyKGso/K30v6c0n/W7iONoSkJ20vVDenH0Y7JP1S0herUdmDtreULqoFByQ90vZBCfcBYntE0lclfS4ifl26niYi4rcR8Se6eNvFd9seypGZ7Q9JWo2IhdK1tORPI+JdkvZJ+nQ11hw2myW9S9I/RsSNks5LmilbUm+q0dJtkr7S9rEJ9wFRzai/Kul4RHytdD29qn5d/q6kvYVLaeo9km6rZtVzkt5n+5/LltRcRKxU/1yV9HVJ7y5bUSPLkpbX/TZ4QhfDfpjtk/RMRLzc9oEJ9wFQnYg8JulcRPxd6Xqasn2N7XdUf79a0gck/bRoUQ1FxJGI2B4R47r4a/N3IuLjhctqxPaW6kS9qjHGByUN3SqziPgPSS/Z3lXter+koVp0cBkH1YeRjFTzNnuDqLqv64SkrbaXJf1VRBwrW1Vj75H0CUmL1bxakv6iuo3hMLlW0peqs/9v0cX76g71EsIkxiR9/eJnCG2W9C8R8c2yJTX2GUnHq3HGzyV9snA9jVX/o/2ApE/15fjDuhQSALAxxjIAkBDhDgAJEe4AkBDhDgAJEe4AkBDhDgAJEe4AkND/AbJqLEFijyXCAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "candidates_new['num_sent'].hist(log=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "id": "97570ba1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>candownload</th>\n",
       "      <th>url_c</th>\n",
       "      <th>url_q</th>\n",
       "      <th>url_n</th>\n",
       "      <th>url_largest</th>\n",
       "      <th>owner</th>\n",
       "      <th>owner_loc</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>...</th>\n",
       "      <th>time</th>\n",
       "      <th>week</th>\n",
       "      <th>unique_week</th>\n",
       "      <th>text_data</th>\n",
       "      <th>Chinese</th>\n",
       "      <th>English</th>\n",
       "      <th>Other_Lang</th>\n",
       "      <th>revised_text</th>\n",
       "      <th>text_len</th>\n",
       "      <th>num_sent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>51859675789</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185967578...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185967578...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185967578...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185967578...</td>\n",
       "      <td>57210421@N04</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>灭渡桥</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2022-02-03 13:33:40</td>\n",
       "      <td>5</td>\n",
       "      <td>202205</td>\n",
       "      <td>. It has title ...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>It has title of Extinction Bridge</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>51858385192</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838519...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838519...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838519...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838519...</td>\n",
       "      <td>57210421@N04</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>下津桥</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2022-02-03 14:24:46</td>\n",
       "      <td>5</td>\n",
       "      <td>202205</td>\n",
       "      <td>. It has title ...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>It has title of Shimotsu Bridge</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4111</th>\n",
       "      <td>27910400829</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/4751/27910400829...</td>\n",
       "      <td>https://live.staticflickr.com/4751/27910400829...</td>\n",
       "      <td>https://live.staticflickr.com/4751/27910400829...</td>\n",
       "      <td>https://live.staticflickr.com/4751/27910400829...</td>\n",
       "      <td>127423136@N05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Corridor, Hanshan Temple - Suzhou, China</td>\n",
       "      <td>OLYMPUS DIGITAL CAMERA</td>\n",
       "      <td>...</td>\n",
       "      <td>2018-01-05 12:08:29</td>\n",
       "      <td>1</td>\n",
       "      <td>201801</td>\n",
       "      <td>OLYMPUS DIGITAL CAMERA. It has title of Corrid...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>It has title of Corridor, Hanshan Temple - Suz...</td>\n",
       "      <td>19</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>51858384372</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838437...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838437...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838437...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838437...</td>\n",
       "      <td>57210421@N04</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>上津桥</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2022-02-03 14:41:14</td>\n",
       "      <td>5</td>\n",
       "      <td>202205</td>\n",
       "      <td>. It has title ...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>It has title of Shangjin Bridge</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4116</th>\n",
       "      <td>38919131931</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/4544/38919131931...</td>\n",
       "      <td>https://live.staticflickr.com/4544/38919131931...</td>\n",
       "      <td>https://live.staticflickr.com/4544/38919131931...</td>\n",
       "      <td>https://live.staticflickr.com/4544/38919131931...</td>\n",
       "      <td>95645163@N00</td>\n",
       "      <td>Milan</td>\n",
       "      <td>IMG_6925</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2017-08-22 13:41:19</td>\n",
       "      <td>34</td>\n",
       "      <td>201734</td>\n",
       "      <td>. It has title of IMG_6925. It has tags of suz...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>It has tags of suzhoushi, jiangsusheng, china, cn</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4073</th>\n",
       "      <td>26221985087</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/885/26221985087_...</td>\n",
       "      <td>https://live.staticflickr.com/885/26221985087_...</td>\n",
       "      <td>https://live.staticflickr.com/885/26221985087_...</td>\n",
       "      <td>https://live.staticflickr.com/885/26221985087_...</td>\n",
       "      <td>152311060@N02</td>\n",
       "      <td>Tuxedo Park, USA</td>\n",
       "      <td>Freshmen go to China</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2018-03-20 11:59:52</td>\n",
       "      <td>12</td>\n",
       "      <td>201812</td>\n",
       "      <td>. It has title of Freshmen go to China. It has...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>It has title of Freshmen go to China</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4078</th>\n",
       "      <td>41140126964</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/948/41140126964_...</td>\n",
       "      <td>https://live.staticflickr.com/948/41140126964_...</td>\n",
       "      <td>https://live.staticflickr.com/948/41140126964_...</td>\n",
       "      <td>https://live.staticflickr.com/948/41140126964_...</td>\n",
       "      <td>7287512@N02</td>\n",
       "      <td>Kokomo, IN, USA</td>\n",
       "      <td>DSC_20180421_10816</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2018-04-21 13:57:48</td>\n",
       "      <td>16</td>\n",
       "      <td>201816</td>\n",
       "      <td>. It has title of DSC_20180421_10816. It has t...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>It has tags of china, places</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4079</th>\n",
       "      <td>41814418522</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/957/41814418522_...</td>\n",
       "      <td>https://live.staticflickr.com/957/41814418522_...</td>\n",
       "      <td>https://live.staticflickr.com/957/41814418522_...</td>\n",
       "      <td>https://live.staticflickr.com/957/41814418522_...</td>\n",
       "      <td>7287512@N02</td>\n",
       "      <td>Kokomo, IN, USA</td>\n",
       "      <td>DSC_20180421_10818</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2018-04-21 13:58:53</td>\n",
       "      <td>16</td>\n",
       "      <td>201816</td>\n",
       "      <td>. It has title of DSC_20180421_10818. It has t...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>It has tags of china, places</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4080</th>\n",
       "      <td>41140126144</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/864/41140126144_...</td>\n",
       "      <td>https://live.staticflickr.com/864/41140126144_...</td>\n",
       "      <td>https://live.staticflickr.com/864/41140126144_...</td>\n",
       "      <td>https://live.staticflickr.com/864/41140126144_...</td>\n",
       "      <td>7287512@N02</td>\n",
       "      <td>Kokomo, IN, USA</td>\n",
       "      <td>DSC_20180421_10825</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2018-04-21 14:01:20</td>\n",
       "      <td>16</td>\n",
       "      <td>201816</td>\n",
       "      <td>. It has title of DSC_20180421_10825. It has t...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>It has tags of china, family, pedro, people, p...</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4082</th>\n",
       "      <td>41857278401</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/958/41857278401_...</td>\n",
       "      <td>https://live.staticflickr.com/958/41857278401_...</td>\n",
       "      <td>https://live.staticflickr.com/958/41857278401_...</td>\n",
       "      <td>https://live.staticflickr.com/958/41857278401_...</td>\n",
       "      <td>7287512@N02</td>\n",
       "      <td>Kokomo, IN, USA</td>\n",
       "      <td>DSC_20180421_10836</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2018-04-21 14:05:25</td>\n",
       "      <td>16</td>\n",
       "      <td>201816</td>\n",
       "      <td>. It has title of DSC_20180421_10836. It has t...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>It has tags of china, places</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>754 rows × 34 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            index  candownload  \\\n",
       "8     51859675789          1.0   \n",
       "13    51858385192          1.0   \n",
       "4111  27910400829          1.0   \n",
       "16    51858384372          1.0   \n",
       "4116  38919131931          1.0   \n",
       "...           ...          ...   \n",
       "4073  26221985087          1.0   \n",
       "4078  41140126964          1.0   \n",
       "4079  41814418522          1.0   \n",
       "4080  41140126144          1.0   \n",
       "4082  41857278401          1.0   \n",
       "\n",
       "                                                  url_c  \\\n",
       "8     https://live.staticflickr.com/65535/5185967578...   \n",
       "13    https://live.staticflickr.com/65535/5185838519...   \n",
       "4111  https://live.staticflickr.com/4751/27910400829...   \n",
       "16    https://live.staticflickr.com/65535/5185838437...   \n",
       "4116  https://live.staticflickr.com/4544/38919131931...   \n",
       "...                                                 ...   \n",
       "4073  https://live.staticflickr.com/885/26221985087_...   \n",
       "4078  https://live.staticflickr.com/948/41140126964_...   \n",
       "4079  https://live.staticflickr.com/957/41814418522_...   \n",
       "4080  https://live.staticflickr.com/864/41140126144_...   \n",
       "4082  https://live.staticflickr.com/958/41857278401_...   \n",
       "\n",
       "                                                  url_q  \\\n",
       "8     https://live.staticflickr.com/65535/5185967578...   \n",
       "13    https://live.staticflickr.com/65535/5185838519...   \n",
       "4111  https://live.staticflickr.com/4751/27910400829...   \n",
       "16    https://live.staticflickr.com/65535/5185838437...   \n",
       "4116  https://live.staticflickr.com/4544/38919131931...   \n",
       "...                                                 ...   \n",
       "4073  https://live.staticflickr.com/885/26221985087_...   \n",
       "4078  https://live.staticflickr.com/948/41140126964_...   \n",
       "4079  https://live.staticflickr.com/957/41814418522_...   \n",
       "4080  https://live.staticflickr.com/864/41140126144_...   \n",
       "4082  https://live.staticflickr.com/958/41857278401_...   \n",
       "\n",
       "                                                  url_n  \\\n",
       "8     https://live.staticflickr.com/65535/5185967578...   \n",
       "13    https://live.staticflickr.com/65535/5185838519...   \n",
       "4111  https://live.staticflickr.com/4751/27910400829...   \n",
       "16    https://live.staticflickr.com/65535/5185838437...   \n",
       "4116  https://live.staticflickr.com/4544/38919131931...   \n",
       "...                                                 ...   \n",
       "4073  https://live.staticflickr.com/885/26221985087_...   \n",
       "4078  https://live.staticflickr.com/948/41140126964_...   \n",
       "4079  https://live.staticflickr.com/957/41814418522_...   \n",
       "4080  https://live.staticflickr.com/864/41140126144_...   \n",
       "4082  https://live.staticflickr.com/958/41857278401_...   \n",
       "\n",
       "                                            url_largest          owner  \\\n",
       "8     https://live.staticflickr.com/65535/5185967578...   57210421@N04   \n",
       "13    https://live.staticflickr.com/65535/5185838519...   57210421@N04   \n",
       "4111  https://live.staticflickr.com/4751/27910400829...  127423136@N05   \n",
       "16    https://live.staticflickr.com/65535/5185838437...   57210421@N04   \n",
       "4116  https://live.staticflickr.com/4544/38919131931...   95645163@N00   \n",
       "...                                                 ...            ...   \n",
       "4073  https://live.staticflickr.com/885/26221985087_...  152311060@N02   \n",
       "4078  https://live.staticflickr.com/948/41140126964_...    7287512@N02   \n",
       "4079  https://live.staticflickr.com/957/41814418522_...    7287512@N02   \n",
       "4080  https://live.staticflickr.com/864/41140126144_...    7287512@N02   \n",
       "4082  https://live.staticflickr.com/958/41857278401_...    7287512@N02   \n",
       "\n",
       "             owner_loc                                     title  \\\n",
       "8       Beijing, China                                       灭渡桥   \n",
       "13      Beijing, China                                       下津桥   \n",
       "4111               NaN  Corridor, Hanshan Temple - Suzhou, China   \n",
       "16      Beijing, China                                       上津桥   \n",
       "4116             Milan                                  IMG_6925   \n",
       "...                ...                                       ...   \n",
       "4073  Tuxedo Park, USA                      Freshmen go to China   \n",
       "4078   Kokomo, IN, USA                        DSC_20180421_10816   \n",
       "4079   Kokomo, IN, USA                        DSC_20180421_10818   \n",
       "4080   Kokomo, IN, USA                        DSC_20180421_10825   \n",
       "4082   Kokomo, IN, USA                        DSC_20180421_10836   \n",
       "\n",
       "                          description  ...                time week  \\\n",
       "8                                      ... 2022-02-03 13:33:40    5   \n",
       "13                                     ... 2022-02-03 14:24:46    5   \n",
       "4111           OLYMPUS DIGITAL CAMERA  ... 2018-01-05 12:08:29    1   \n",
       "16                                     ... 2022-02-03 14:41:14    5   \n",
       "4116                                   ... 2017-08-22 13:41:19   34   \n",
       "...                               ...  ...                 ...  ...   \n",
       "4073                                   ... 2018-03-20 11:59:52   12   \n",
       "4078                                   ... 2018-04-21 13:57:48   16   \n",
       "4079                                   ... 2018-04-21 13:58:53   16   \n",
       "4080                                   ... 2018-04-21 14:01:20   16   \n",
       "4082                                   ... 2018-04-21 14:05:25   16   \n",
       "\n",
       "      unique_week                                          text_data Chinese  \\\n",
       "8          202205                                 . It has title ...       1   \n",
       "13         202205                                 . It has title ...       1   \n",
       "4111       201801  OLYMPUS DIGITAL CAMERA. It has title of Corrid...       0   \n",
       "16         202205                                 . It has title ...       1   \n",
       "4116       201734  . It has title of IMG_6925. It has tags of suz...       0   \n",
       "...           ...                                                ...     ...   \n",
       "4073       201812  . It has title of Freshmen go to China. It has...       0   \n",
       "4078       201816  . It has title of DSC_20180421_10816. It has t...       0   \n",
       "4079       201816  . It has title of DSC_20180421_10818. It has t...       0   \n",
       "4080       201816  . It has title of DSC_20180421_10825. It has t...       0   \n",
       "4082       201816  . It has title of DSC_20180421_10836. It has t...       0   \n",
       "\n",
       "      English  Other_Lang                                       revised_text  \\\n",
       "8           0           0                  It has title of Extinction Bridge   \n",
       "13          0           0                    It has title of Shimotsu Bridge   \n",
       "4111        1           0  It has title of Corridor, Hanshan Temple - Suz...   \n",
       "16          0           0                    It has title of Shangjin Bridge   \n",
       "4116        0           1  It has tags of suzhoushi, jiangsusheng, china, cn   \n",
       "...       ...         ...                                                ...   \n",
       "4073        1           0               It has title of Freshmen go to China   \n",
       "4078        0           1                       It has tags of china, places   \n",
       "4079        0           1                       It has tags of china, places   \n",
       "4080        1           0  It has tags of china, family, pedro, people, p...   \n",
       "4082        0           1                       It has tags of china, places   \n",
       "\n",
       "     text_len  num_sent  \n",
       "8           6         1  \n",
       "13          6         1  \n",
       "4111       19         2  \n",
       "16          6         1  \n",
       "4116        8         1  \n",
       "...       ...       ...  \n",
       "4073        8         1  \n",
       "4078        6         1  \n",
       "4079        6         1  \n",
       "4080        9         1  \n",
       "4082        6         1  \n",
       "\n",
       "[754 rows x 34 columns]"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "candidates_new"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e84ffe48",
   "metadata": {},
   "source": [
    "#### Merging the Revised Text data back to the full Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d0b6a5f3",
   "metadata": {},
   "source": [
    "Merging first back to the subset with text data and copy revised text data to the duplicates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "id": "1f13d888",
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_to_use = candidates_new.columns.difference(df_new.columns)\n",
    "new = df_new.merge(candidates_new[cols_to_use], left_index=True, right_index=True, how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "id": "5cd43871",
   "metadata": {},
   "outputs": [],
   "source": [
    "text_dict = {}\n",
    "for i in range(len(new)):\n",
    "    if new.iloc[i]['text_data'] not in text_dict:\n",
    "        text_dict[new.iloc[i]['text_data']] = new.iloc[i]['revised_text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "id": "05a77b84",
   "metadata": {},
   "outputs": [],
   "source": [
    "new['revised_text'] = new['text_data'].apply(lambda x: text_dict[x])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6475fa6",
   "metadata": {},
   "source": [
    "1405 data points have a text data field, but do not have a revised text data field, since the texts they have are *invalid*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "id": "b615d3f6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>candownload</th>\n",
       "      <th>url_c</th>\n",
       "      <th>url_q</th>\n",
       "      <th>url_n</th>\n",
       "      <th>url_largest</th>\n",
       "      <th>owner</th>\n",
       "      <th>owner_loc</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>...</th>\n",
       "      <th>time</th>\n",
       "      <th>week</th>\n",
       "      <th>unique_week</th>\n",
       "      <th>text_data</th>\n",
       "      <th>Chinese</th>\n",
       "      <th>English</th>\n",
       "      <th>Other_Lang</th>\n",
       "      <th>num_sent</th>\n",
       "      <th>revised_text</th>\n",
       "      <th>text_len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>125</th>\n",
       "      <td>51836197137</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5183619713...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5183619713...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5183619713...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5183619713...</td>\n",
       "      <td>193769199@N05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>43B06C03-0A8C-4670-AA3D-1BF034F6B322_1_105_c</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2021-03-20 20:10:55</td>\n",
       "      <td>11</td>\n",
       "      <td>202111</td>\n",
       "      <td>. It has title of 43B06C03-0A8C-4670-AA3D-1BF0...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>51837499654</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5183749965...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5183749965...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5183749965...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5183749965...</td>\n",
       "      <td>193769199@N05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>936635AE-45E9-4078-9DC8-4854241288C2_1_105_c</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2021-03-20 18:19:52</td>\n",
       "      <td>11</td>\n",
       "      <td>202111</td>\n",
       "      <td>. It has title of 936635AE-45E9-4078-9DC8-4854...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>127</th>\n",
       "      <td>51837879515</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5183787951...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5183787951...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5183787951...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5183787951...</td>\n",
       "      <td>193769199@N05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4894AD62-54A5-42C3-8BD6-9EDB9C1D0414_1_105_c</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2021-03-20 18:16:18</td>\n",
       "      <td>11</td>\n",
       "      <td>202111</td>\n",
       "      <td>. It has title of 4894AD62-54A5-42C3-8BD6-9EDB...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128</th>\n",
       "      <td>51837147936</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5183714793...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5183714793...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5183714793...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5183714793...</td>\n",
       "      <td>193769199@N05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3F886916-F7F9-4723-82DE-20F3C4DF2C40_1_105_c</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2021-03-20 17:18:39</td>\n",
       "      <td>11</td>\n",
       "      <td>202111</td>\n",
       "      <td>. It has title of 3F886916-F7F9-4723-82DE-20F3...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129</th>\n",
       "      <td>51837879165</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5183787916...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5183787916...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5183787916...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5183787916...</td>\n",
       "      <td>193769199@N05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>057F3E9D-C4AA-4454-A214-31EF82EB31AB_1_105_c</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2021-03-20 15:58:25</td>\n",
       "      <td>11</td>\n",
       "      <td>202111</td>\n",
       "      <td>. It has title of 057F3E9D-C4AA-4454-A214-31EF...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4224</th>\n",
       "      <td>38397950826</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/4578/38397950826...</td>\n",
       "      <td>https://live.staticflickr.com/4578/38397950826...</td>\n",
       "      <td>https://live.staticflickr.com/4578/38397950826...</td>\n",
       "      <td>https://live.staticflickr.com/4578/38397950826...</td>\n",
       "      <td>28571579@N03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20171112_IMG_0514</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2017-11-12 19:32:22</td>\n",
       "      <td>45</td>\n",
       "      <td>201745</td>\n",
       "      <td>. It has title of 20171112_IMG_0514. It has ta...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4225</th>\n",
       "      <td>38397950426</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/4582/38397950426...</td>\n",
       "      <td>https://live.staticflickr.com/4582/38397950426...</td>\n",
       "      <td>https://live.staticflickr.com/4582/38397950426...</td>\n",
       "      <td>https://live.staticflickr.com/4582/38397950426...</td>\n",
       "      <td>28571579@N03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20171112_IMG_0515</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2017-11-12 19:33:08</td>\n",
       "      <td>45</td>\n",
       "      <td>201745</td>\n",
       "      <td>. It has title of 20171112_IMG_0515. It has ta...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4226</th>\n",
       "      <td>38397948496</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/4517/38397948496...</td>\n",
       "      <td>https://live.staticflickr.com/4517/38397948496...</td>\n",
       "      <td>https://live.staticflickr.com/4517/38397948496...</td>\n",
       "      <td>https://live.staticflickr.com/4517/38397948496...</td>\n",
       "      <td>28571579@N03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20171112_IMG_0518</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2017-11-12 19:33:52</td>\n",
       "      <td>45</td>\n",
       "      <td>201745</td>\n",
       "      <td>. It has title of 20171112_IMG_0518. It has ta...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4227</th>\n",
       "      <td>38454117381</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/4552/38454117381...</td>\n",
       "      <td>https://live.staticflickr.com/4552/38454117381...</td>\n",
       "      <td>https://live.staticflickr.com/4552/38454117381...</td>\n",
       "      <td>https://live.staticflickr.com/4552/38454117381...</td>\n",
       "      <td>28571579@N03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20171112_IMG_0531</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2017-11-12 19:40:39</td>\n",
       "      <td>45</td>\n",
       "      <td>201745</td>\n",
       "      <td>. It has title of 20171112_IMG_0531. It has ta...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4228</th>\n",
       "      <td>38397940256</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/4576/38397940256...</td>\n",
       "      <td>https://live.staticflickr.com/4576/38397940256...</td>\n",
       "      <td>https://live.staticflickr.com/4576/38397940256...</td>\n",
       "      <td>https://live.staticflickr.com/4576/38397940256...</td>\n",
       "      <td>28571579@N03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20171112_IMG_0539</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2017-11-12 19:45:07</td>\n",
       "      <td>45</td>\n",
       "      <td>201745</td>\n",
       "      <td>. It has title of 20171112_IMG_0539. It has ta...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1405 rows × 34 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            index  candownload  \\\n",
       "125   51836197137          1.0   \n",
       "126   51837499654          1.0   \n",
       "127   51837879515          1.0   \n",
       "128   51837147936          1.0   \n",
       "129   51837879165          1.0   \n",
       "...           ...          ...   \n",
       "4224  38397950826          1.0   \n",
       "4225  38397950426          1.0   \n",
       "4226  38397948496          1.0   \n",
       "4227  38454117381          1.0   \n",
       "4228  38397940256          1.0   \n",
       "\n",
       "                                                  url_c  \\\n",
       "125   https://live.staticflickr.com/65535/5183619713...   \n",
       "126   https://live.staticflickr.com/65535/5183749965...   \n",
       "127   https://live.staticflickr.com/65535/5183787951...   \n",
       "128   https://live.staticflickr.com/65535/5183714793...   \n",
       "129   https://live.staticflickr.com/65535/5183787916...   \n",
       "...                                                 ...   \n",
       "4224  https://live.staticflickr.com/4578/38397950826...   \n",
       "4225  https://live.staticflickr.com/4582/38397950426...   \n",
       "4226  https://live.staticflickr.com/4517/38397948496...   \n",
       "4227  https://live.staticflickr.com/4552/38454117381...   \n",
       "4228  https://live.staticflickr.com/4576/38397940256...   \n",
       "\n",
       "                                                  url_q  \\\n",
       "125   https://live.staticflickr.com/65535/5183619713...   \n",
       "126   https://live.staticflickr.com/65535/5183749965...   \n",
       "127   https://live.staticflickr.com/65535/5183787951...   \n",
       "128   https://live.staticflickr.com/65535/5183714793...   \n",
       "129   https://live.staticflickr.com/65535/5183787916...   \n",
       "...                                                 ...   \n",
       "4224  https://live.staticflickr.com/4578/38397950826...   \n",
       "4225  https://live.staticflickr.com/4582/38397950426...   \n",
       "4226  https://live.staticflickr.com/4517/38397948496...   \n",
       "4227  https://live.staticflickr.com/4552/38454117381...   \n",
       "4228  https://live.staticflickr.com/4576/38397940256...   \n",
       "\n",
       "                                                  url_n  \\\n",
       "125   https://live.staticflickr.com/65535/5183619713...   \n",
       "126   https://live.staticflickr.com/65535/5183749965...   \n",
       "127   https://live.staticflickr.com/65535/5183787951...   \n",
       "128   https://live.staticflickr.com/65535/5183714793...   \n",
       "129   https://live.staticflickr.com/65535/5183787916...   \n",
       "...                                                 ...   \n",
       "4224  https://live.staticflickr.com/4578/38397950826...   \n",
       "4225  https://live.staticflickr.com/4582/38397950426...   \n",
       "4226  https://live.staticflickr.com/4517/38397948496...   \n",
       "4227  https://live.staticflickr.com/4552/38454117381...   \n",
       "4228  https://live.staticflickr.com/4576/38397940256...   \n",
       "\n",
       "                                            url_largest          owner  \\\n",
       "125   https://live.staticflickr.com/65535/5183619713...  193769199@N05   \n",
       "126   https://live.staticflickr.com/65535/5183749965...  193769199@N05   \n",
       "127   https://live.staticflickr.com/65535/5183787951...  193769199@N05   \n",
       "128   https://live.staticflickr.com/65535/5183714793...  193769199@N05   \n",
       "129   https://live.staticflickr.com/65535/5183787916...  193769199@N05   \n",
       "...                                                 ...            ...   \n",
       "4224  https://live.staticflickr.com/4578/38397950826...   28571579@N03   \n",
       "4225  https://live.staticflickr.com/4582/38397950426...   28571579@N03   \n",
       "4226  https://live.staticflickr.com/4517/38397948496...   28571579@N03   \n",
       "4227  https://live.staticflickr.com/4552/38454117381...   28571579@N03   \n",
       "4228  https://live.staticflickr.com/4576/38397940256...   28571579@N03   \n",
       "\n",
       "     owner_loc                                         title description  ...  \\\n",
       "125        NaN  43B06C03-0A8C-4670-AA3D-1BF034F6B322_1_105_c              ...   \n",
       "126        NaN  936635AE-45E9-4078-9DC8-4854241288C2_1_105_c              ...   \n",
       "127        NaN  4894AD62-54A5-42C3-8BD6-9EDB9C1D0414_1_105_c              ...   \n",
       "128        NaN  3F886916-F7F9-4723-82DE-20F3C4DF2C40_1_105_c              ...   \n",
       "129        NaN  057F3E9D-C4AA-4454-A214-31EF82EB31AB_1_105_c              ...   \n",
       "...        ...                                           ...         ...  ...   \n",
       "4224       NaN                             20171112_IMG_0514              ...   \n",
       "4225       NaN                             20171112_IMG_0515              ...   \n",
       "4226       NaN                             20171112_IMG_0518              ...   \n",
       "4227       NaN                             20171112_IMG_0531              ...   \n",
       "4228       NaN                             20171112_IMG_0539              ...   \n",
       "\n",
       "                    time week  unique_week  \\\n",
       "125  2021-03-20 20:10:55   11       202111   \n",
       "126  2021-03-20 18:19:52   11       202111   \n",
       "127  2021-03-20 18:16:18   11       202111   \n",
       "128  2021-03-20 17:18:39   11       202111   \n",
       "129  2021-03-20 15:58:25   11       202111   \n",
       "...                  ...  ...          ...   \n",
       "4224 2017-11-12 19:32:22   45       201745   \n",
       "4225 2017-11-12 19:33:08   45       201745   \n",
       "4226 2017-11-12 19:33:52   45       201745   \n",
       "4227 2017-11-12 19:40:39   45       201745   \n",
       "4228 2017-11-12 19:45:07   45       201745   \n",
       "\n",
       "                                              text_data Chinese  English  \\\n",
       "125   . It has title of 43B06C03-0A8C-4670-AA3D-1BF0...     NaN      NaN   \n",
       "126   . It has title of 936635AE-45E9-4078-9DC8-4854...     NaN      NaN   \n",
       "127   . It has title of 4894AD62-54A5-42C3-8BD6-9EDB...     NaN      NaN   \n",
       "128   . It has title of 3F886916-F7F9-4723-82DE-20F3...     NaN      NaN   \n",
       "129   . It has title of 057F3E9D-C4AA-4454-A214-31EF...     NaN      NaN   \n",
       "...                                                 ...     ...      ...   \n",
       "4224  . It has title of 20171112_IMG_0514. It has ta...     NaN      NaN   \n",
       "4225  . It has title of 20171112_IMG_0515. It has ta...     NaN      NaN   \n",
       "4226  . It has title of 20171112_IMG_0518. It has ta...     NaN      NaN   \n",
       "4227  . It has title of 20171112_IMG_0531. It has ta...     NaN      NaN   \n",
       "4228  . It has title of 20171112_IMG_0539. It has ta...     NaN      NaN   \n",
       "\n",
       "      Other_Lang  num_sent revised_text  text_len  \n",
       "125          NaN       NaN          NaN       NaN  \n",
       "126          NaN       NaN          NaN       NaN  \n",
       "127          NaN       NaN          NaN       NaN  \n",
       "128          NaN       NaN          NaN       NaN  \n",
       "129          NaN       NaN          NaN       NaN  \n",
       "...          ...       ...          ...       ...  \n",
       "4224         NaN       NaN          NaN       NaN  \n",
       "4225         NaN       NaN          NaN       NaN  \n",
       "4226         NaN       NaN          NaN       NaN  \n",
       "4227         NaN       NaN          NaN       NaN  \n",
       "4228         NaN       NaN          NaN       NaN  \n",
       "\n",
       "[1405 rows x 34 columns]"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new[new['revised_text'].isna()]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1265d8f0",
   "metadata": {},
   "source": [
    "Merging then back to the full dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "f6218466",
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_to_use = new.columns.difference(df.columns)\n",
    "final_df = df.merge(new[cols_to_use], left_index=True, right_index=True, how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "b4f81bc9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>candownload</th>\n",
       "      <th>url_c</th>\n",
       "      <th>url_q</th>\n",
       "      <th>url_n</th>\n",
       "      <th>url_largest</th>\n",
       "      <th>owner</th>\n",
       "      <th>owner_loc</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>...</th>\n",
       "      <th>time</th>\n",
       "      <th>week</th>\n",
       "      <th>unique_week</th>\n",
       "      <th>Chinese</th>\n",
       "      <th>English</th>\n",
       "      <th>Other_Lang</th>\n",
       "      <th>num_sent</th>\n",
       "      <th>revised_text</th>\n",
       "      <th>text_data</th>\n",
       "      <th>text_len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>51859675789</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185967578...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185967578...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185967578...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185967578...</td>\n",
       "      <td>57210421@N04</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>灭渡桥</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2022-02-03 13:33:40</td>\n",
       "      <td>5</td>\n",
       "      <td>202205</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>It has title of Extinction Bridge</td>\n",
       "      <td>. It has title ...</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>51859339946</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185933994...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185933994...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185933994...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185933994...</td>\n",
       "      <td>57210421@N04</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>灭渡桥</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2022-02-03 13:34:07</td>\n",
       "      <td>5</td>\n",
       "      <td>202205</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>It has title of Extinction Bridge</td>\n",
       "      <td>. It has title ...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>51858385807</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838580...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838580...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838580...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838580...</td>\n",
       "      <td>57210421@N04</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>灭渡桥</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2022-02-03 13:34:18</td>\n",
       "      <td>5</td>\n",
       "      <td>202205</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>It has title of Extinction Bridge</td>\n",
       "      <td>. It has title ...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>51859339546</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185933954...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185933954...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185933954...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185933954...</td>\n",
       "      <td>57210421@N04</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>灭渡桥</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2022-02-03 13:34:52</td>\n",
       "      <td>5</td>\n",
       "      <td>202205</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>It has title of Extinction Bridge</td>\n",
       "      <td>. It has title ...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>51858385377</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838537...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838537...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838537...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5185838537...</td>\n",
       "      <td>57210421@N04</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>灭渡桥</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2022-02-03 13:38:02</td>\n",
       "      <td>5</td>\n",
       "      <td>202205</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>It has title of Extinction Bridge</td>\n",
       "      <td>. It has title ...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4225</th>\n",
       "      <td>38397950426</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/4582/38397950426...</td>\n",
       "      <td>https://live.staticflickr.com/4582/38397950426...</td>\n",
       "      <td>https://live.staticflickr.com/4582/38397950426...</td>\n",
       "      <td>https://live.staticflickr.com/4582/38397950426...</td>\n",
       "      <td>28571579@N03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20171112_IMG_0515</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2017-11-12 19:33:08</td>\n",
       "      <td>45</td>\n",
       "      <td>201745</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>. It has title of 20171112_IMG_0515. It has ta...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4226</th>\n",
       "      <td>38397948496</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/4517/38397948496...</td>\n",
       "      <td>https://live.staticflickr.com/4517/38397948496...</td>\n",
       "      <td>https://live.staticflickr.com/4517/38397948496...</td>\n",
       "      <td>https://live.staticflickr.com/4517/38397948496...</td>\n",
       "      <td>28571579@N03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20171112_IMG_0518</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2017-11-12 19:33:52</td>\n",
       "      <td>45</td>\n",
       "      <td>201745</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>. It has title of 20171112_IMG_0518. It has ta...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4227</th>\n",
       "      <td>38454117381</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/4552/38454117381...</td>\n",
       "      <td>https://live.staticflickr.com/4552/38454117381...</td>\n",
       "      <td>https://live.staticflickr.com/4552/38454117381...</td>\n",
       "      <td>https://live.staticflickr.com/4552/38454117381...</td>\n",
       "      <td>28571579@N03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20171112_IMG_0531</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2017-11-12 19:40:39</td>\n",
       "      <td>45</td>\n",
       "      <td>201745</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>. It has title of 20171112_IMG_0531. It has ta...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4228</th>\n",
       "      <td>38397940256</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/4576/38397940256...</td>\n",
       "      <td>https://live.staticflickr.com/4576/38397940256...</td>\n",
       "      <td>https://live.staticflickr.com/4576/38397940256...</td>\n",
       "      <td>https://live.staticflickr.com/4576/38397940256...</td>\n",
       "      <td>28571579@N03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20171112_IMG_0539</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2017-11-12 19:45:07</td>\n",
       "      <td>45</td>\n",
       "      <td>201745</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>. It has title of 20171112_IMG_0539. It has ta...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4229</th>\n",
       "      <td>50150116276</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5015011627...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5015011627...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5015011627...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5015011627...</td>\n",
       "      <td>57210421@N04</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>吴中博物馆</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2020-07-18 15:49:45</td>\n",
       "      <td>29</td>\n",
       "      <td>202029</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>It has title of Wuzhong Museum</td>\n",
       "      <td>. It has title ...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3137 rows × 34 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            index  candownload  \\\n",
       "8     51859675789          1.0   \n",
       "9     51859339946          1.0   \n",
       "10    51858385807          1.0   \n",
       "11    51859339546          1.0   \n",
       "12    51858385377          1.0   \n",
       "...           ...          ...   \n",
       "4225  38397950426          1.0   \n",
       "4226  38397948496          1.0   \n",
       "4227  38454117381          1.0   \n",
       "4228  38397940256          1.0   \n",
       "4229  50150116276          1.0   \n",
       "\n",
       "                                                  url_c  \\\n",
       "8     https://live.staticflickr.com/65535/5185967578...   \n",
       "9     https://live.staticflickr.com/65535/5185933994...   \n",
       "10    https://live.staticflickr.com/65535/5185838580...   \n",
       "11    https://live.staticflickr.com/65535/5185933954...   \n",
       "12    https://live.staticflickr.com/65535/5185838537...   \n",
       "...                                                 ...   \n",
       "4225  https://live.staticflickr.com/4582/38397950426...   \n",
       "4226  https://live.staticflickr.com/4517/38397948496...   \n",
       "4227  https://live.staticflickr.com/4552/38454117381...   \n",
       "4228  https://live.staticflickr.com/4576/38397940256...   \n",
       "4229  https://live.staticflickr.com/65535/5015011627...   \n",
       "\n",
       "                                                  url_q  \\\n",
       "8     https://live.staticflickr.com/65535/5185967578...   \n",
       "9     https://live.staticflickr.com/65535/5185933994...   \n",
       "10    https://live.staticflickr.com/65535/5185838580...   \n",
       "11    https://live.staticflickr.com/65535/5185933954...   \n",
       "12    https://live.staticflickr.com/65535/5185838537...   \n",
       "...                                                 ...   \n",
       "4225  https://live.staticflickr.com/4582/38397950426...   \n",
       "4226  https://live.staticflickr.com/4517/38397948496...   \n",
       "4227  https://live.staticflickr.com/4552/38454117381...   \n",
       "4228  https://live.staticflickr.com/4576/38397940256...   \n",
       "4229  https://live.staticflickr.com/65535/5015011627...   \n",
       "\n",
       "                                                  url_n  \\\n",
       "8     https://live.staticflickr.com/65535/5185967578...   \n",
       "9     https://live.staticflickr.com/65535/5185933994...   \n",
       "10    https://live.staticflickr.com/65535/5185838580...   \n",
       "11    https://live.staticflickr.com/65535/5185933954...   \n",
       "12    https://live.staticflickr.com/65535/5185838537...   \n",
       "...                                                 ...   \n",
       "4225  https://live.staticflickr.com/4582/38397950426...   \n",
       "4226  https://live.staticflickr.com/4517/38397948496...   \n",
       "4227  https://live.staticflickr.com/4552/38454117381...   \n",
       "4228  https://live.staticflickr.com/4576/38397940256...   \n",
       "4229  https://live.staticflickr.com/65535/5015011627...   \n",
       "\n",
       "                                            url_largest         owner  \\\n",
       "8     https://live.staticflickr.com/65535/5185967578...  57210421@N04   \n",
       "9     https://live.staticflickr.com/65535/5185933994...  57210421@N04   \n",
       "10    https://live.staticflickr.com/65535/5185838580...  57210421@N04   \n",
       "11    https://live.staticflickr.com/65535/5185933954...  57210421@N04   \n",
       "12    https://live.staticflickr.com/65535/5185838537...  57210421@N04   \n",
       "...                                                 ...           ...   \n",
       "4225  https://live.staticflickr.com/4582/38397950426...  28571579@N03   \n",
       "4226  https://live.staticflickr.com/4517/38397948496...  28571579@N03   \n",
       "4227  https://live.staticflickr.com/4552/38454117381...  28571579@N03   \n",
       "4228  https://live.staticflickr.com/4576/38397940256...  28571579@N03   \n",
       "4229  https://live.staticflickr.com/65535/5015011627...  57210421@N04   \n",
       "\n",
       "           owner_loc              title                      description  ...  \\\n",
       "8     Beijing, China                灭渡桥                                   ...   \n",
       "9     Beijing, China                灭渡桥                                   ...   \n",
       "10    Beijing, China                灭渡桥                                   ...   \n",
       "11    Beijing, China                灭渡桥                                   ...   \n",
       "12    Beijing, China                灭渡桥                                   ...   \n",
       "...              ...                ...                              ...  ...   \n",
       "4225             NaN  20171112_IMG_0515                              NaN  ...   \n",
       "4226             NaN  20171112_IMG_0518                              NaN  ...   \n",
       "4227             NaN  20171112_IMG_0531                              NaN  ...   \n",
       "4228             NaN  20171112_IMG_0539                              NaN  ...   \n",
       "4229  Beijing, China              吴中博物馆                                   ...   \n",
       "\n",
       "                    time week  unique_week  Chinese English  Other_Lang  \\\n",
       "8    2022-02-03 13:33:40    5       202205      1.0     0.0         0.0   \n",
       "9    2022-02-03 13:34:07    5       202205      NaN     NaN         NaN   \n",
       "10   2022-02-03 13:34:18    5       202205      NaN     NaN         NaN   \n",
       "11   2022-02-03 13:34:52    5       202205      NaN     NaN         NaN   \n",
       "12   2022-02-03 13:38:02    5       202205      NaN     NaN         NaN   \n",
       "...                  ...  ...          ...      ...     ...         ...   \n",
       "4225 2017-11-12 19:33:08   45       201745      NaN     NaN         NaN   \n",
       "4226 2017-11-12 19:33:52   45       201745      NaN     NaN         NaN   \n",
       "4227 2017-11-12 19:40:39   45       201745      NaN     NaN         NaN   \n",
       "4228 2017-11-12 19:45:07   45       201745      NaN     NaN         NaN   \n",
       "4229 2020-07-18 15:49:45   29       202029      NaN     NaN         NaN   \n",
       "\n",
       "      num_sent                       revised_text  \\\n",
       "8          1.0  It has title of Extinction Bridge   \n",
       "9          NaN  It has title of Extinction Bridge   \n",
       "10         NaN  It has title of Extinction Bridge   \n",
       "11         NaN  It has title of Extinction Bridge   \n",
       "12         NaN  It has title of Extinction Bridge   \n",
       "...        ...                                ...   \n",
       "4225       NaN                                NaN   \n",
       "4226       NaN                                NaN   \n",
       "4227       NaN                                NaN   \n",
       "4228       NaN                                NaN   \n",
       "4229       NaN     It has title of Wuzhong Museum   \n",
       "\n",
       "                                              text_data  text_len  \n",
       "8                                    . It has title ...       6.0  \n",
       "9                                    . It has title ...       NaN  \n",
       "10                                   . It has title ...       NaN  \n",
       "11                                   . It has title ...       NaN  \n",
       "12                                   . It has title ...       NaN  \n",
       "...                                                 ...       ...  \n",
       "4225  . It has title of 20171112_IMG_0515. It has ta...       NaN  \n",
       "4226  . It has title of 20171112_IMG_0518. It has ta...       NaN  \n",
       "4227  . It has title of 20171112_IMG_0531. It has ta...       NaN  \n",
       "4228  . It has title of 20171112_IMG_0539. It has ta...       NaN  \n",
       "4229                                 . It has title ...       NaN  \n",
       "\n",
       "[3137 rows x 34 columns]"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f4143af",
   "metadata": {},
   "source": [
    "A few cases with super long text descriptions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "id": "cae7051e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"The Master of the Nets Garden (Chinese: 网师园) in Suzhou is among the finest gardens in China. It is recognized with other classical Suzhou gardens as a UNESCO World Heritage Site. The garden demonstrates Chinese garden designers' adept skills for synthesizing art, nature, and architecture to create unique metaphysical masterpieces. The Master of the Nets is particularly regarded among garden connoisseurs for its mastering the techniques of relative dimension, contrast, foil, sequence and depth, and borrowed scenery. It has title of Master of the Nets Garden 网师园. It has tags of sony vsco\",\n",
       " 'Suzhou, with its many canals and bridges, is often nicknamed the .Venice of the East. (although many other cities often also claim this title as well). Being very close to Shanghai has allowed it to become of the fastest growing cities in Mainland China. It is also a popular tourist destination with its many UNESCO sites and attractions, such as this one near Tiger Hill.. It has title of Reflections of Suzhou. It has tags of jiangsu, december, building, asia, east, water, city, venice, photography, hill, province, landscape, canal, chinese, peoples, suzhou, river, tiger, house, shanghai, asian, republic, prc, china, peoplesrepublicofchina',\n",
       " 'Suzhou, with its many canals and bridges, is often nicknamed the .Venice of the East. (although many other cities often also claim this title as well). Being very close to Shanghai has allowed it to become of the fastest growing cities in Mainland China. It is also a popular tourist destination with its many UNESCO sites and attractions, such as this one near Tiger Hill.. It has title of The many canals of Suzhou. It has tags of jiangsu, december, building, asia, east, water, city, venice, photography, hill, province, landscape, canal, chinese, peoples, suzhou, river, tiger, house, shanghai, asian, republic, prc, china, peoplesrepublicofchina',\n",
       " \"literally 'Cold Mountain Temple' is well known because of the poem .A Night Mooring by Maple Bridge., by Tang Dynasty poet, Zhang Ji  月落烏啼霜滿天， 江楓漁火對愁眠. The moon descends, crows caw, frostiness fills the sky. riverside maples, fishermen's lights, facing me in my worried sleep. Outside Gusu city lies Cold Mountain Temple. at midnight, the sound of bells reaches the ferries.. It has title of Hanshan Temple. It has tags of suzhou, jiangsu, jiangnan, china, xt, fuji, fujifilm, fujifilmxf\"]"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_df[final_df['num_sent']>5]['revised_text'].tolist()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9b0a2c9d",
   "metadata": {},
   "source": [
    "### Creating a one-hot Embedding for frequent Tags"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab25e418",
   "metadata": {},
   "source": [
    "#### Generating a tag dictionary with appearance counts"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d95282eb",
   "metadata": {},
   "source": [
    "With the tags of the complete dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "id": "e24d3082",
   "metadata": {},
   "outputs": [],
   "source": [
    "tag_dict = {}\n",
    "for i in final_df.index:\n",
    "    list_now = final_df.loc[i]['tag_list']\n",
    "    for w in list_now:\n",
    "        if not w in tag_dict:\n",
    "            tag_dict[w] = {}\n",
    "            tag_dict[w]['count'] = 1\n",
    "            tag_dict[w]['origin'] = [i]\n",
    "        else:\n",
    "            tag_dict[w]['count'] += 1\n",
    "            tag_dict[w]['origin'].append(i)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8de5328",
   "metadata": {},
   "source": [
    "Or alternatively, with the tags from the subset of unique and valid text data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "id": "63549b97",
   "metadata": {},
   "outputs": [],
   "source": [
    "tag_dict = {}\n",
    "for i in candidates_new.index:\n",
    "    list_now = candidates_new.loc[i]['tag_list']\n",
    "    for w in list_now:\n",
    "        if not w in tag_dict:\n",
    "            tag_dict[w] = {}\n",
    "            tag_dict[w]['count'] = 1\n",
    "            tag_dict[w]['origin'] = [i]\n",
    "        else:\n",
    "            tag_dict[w]['count'] += 1\n",
    "            tag_dict[w]['origin'].append(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "id": "d7b0f74c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "339"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tag_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cb14e82e",
   "metadata": {},
   "source": [
    "#### Converting tags to non-capitalized and translating tag words into English, if possible"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "id": "f77b963c",
   "metadata": {},
   "outputs": [],
   "source": [
    "tag_dict_en = {}\n",
    "for k, v in tag_dict.items():\n",
    "    w = GoogleTranslator(source='auto', target='en').translate(k.lower())\n",
    "    if not w in tag_dict_en:\n",
    "        tag_dict_en[w] = {}\n",
    "        tag_dict_en[w]['count'] = v['count']\n",
    "        tag_dict_en[w]['origin'] = set(v['origin'])\n",
    "    else:\n",
    "        tag_dict_en[w]['count'] += v['count']\n",
    "        tag_dict_en[w]['origin'] = tag_dict_en[w]['origin'].union(set(v['origin']))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f7bad01",
   "metadata": {},
   "source": [
    "336 out of 339 tags remained after translating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "id": "f82c62b1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(339, 336)"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tag_dict),len(tag_dict_en)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9b75a68d",
   "metadata": {},
   "source": [
    "#### Converting the tag dictionary to a pandas DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "id": "1c4679dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "tag_df = pd.DataFrame(tag_dict_en).T.sort_values(by='count',ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6e6d7b7c",
   "metadata": {},
   "source": [
    "It could be observed that this tag distribution has a very long tail, and the most frequent tags are only referring to the place, which may need to be filtered out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "id": "295dcd4a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>origin</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>china</th>\n",
       "      <td>672</td>\n",
       "      <td>{4111, 4116, 4117, 2099, 2115, 2116, 2117, 211...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Suzhou</th>\n",
       "      <td>543</td>\n",
       "      <td>{4111, 108, 2901, 349, 350, 351, 2439, 2442, 3...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>jiangsu</th>\n",
       "      <td>293</td>\n",
       "      <td>{3615, 3616, 3617, 3618, 3619, 3620, 3621, 550...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>peoplesrepublicofchina</th>\n",
       "      <td>217</td>\n",
       "      <td>{3615, 3616, 3617, 3618, 3619, 3620, 3621, 550...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>garden</th>\n",
       "      <td>173</td>\n",
       "      <td>{3615, 3616, 4129, 3617, 3618, 3619, 3620, 362...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unescoworldheritage</th>\n",
       "      <td>144</td>\n",
       "      <td>{3615, 3616, 3617, 3618, 3619, 3620, 3621, 362...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>classicalgardens</th>\n",
       "      <td>133</td>\n",
       "      <td>{3615, 3616, 3617, 3618, 3619, 3620, 3621, 362...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>suzhoushi</th>\n",
       "      <td>60</td>\n",
       "      <td>{768, 781, 784, 785, 658, 787, 660, 4116, 4117...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>jiangsusheng</th>\n",
       "      <td>60</td>\n",
       "      <td>{768, 781, 784, 785, 658, 787, 660, 4116, 4117...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>places</th>\n",
       "      <td>60</td>\n",
       "      <td>{3840, 3841, 3842, 3843, 3844, 3845, 3846, 384...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gardenofthehumbleadministrator</th>\n",
       "      <td>54</td>\n",
       "      <td>{3242, 3243, 3244, 3245, 3246, 3247, 3248, 324...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>humble</th>\n",
       "      <td>44</td>\n",
       "      <td>{768, 1908, 2819, 1909, 1910, 395, 781, 1911, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>administrator</th>\n",
       "      <td>41</td>\n",
       "      <td>{768, 1909, 1910, 781, 1911, 784, 785, 658, 78...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cn</th>\n",
       "      <td>33</td>\n",
       "      <td>{4116, 4117, 3484, 3485, 3486, 3487, 3488, 348...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zhu</th>\n",
       "      <td>32</td>\n",
       "      <td>{768, 781, 784, 785, 658, 787, 660, 788, 789, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lingering</th>\n",
       "      <td>31</td>\n",
       "      <td>{653, 783, 677, 678, 679, 680, 681, 682, 683, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>at the</th>\n",
       "      <td>31</td>\n",
       "      <td>{653, 783, 677, 678, 679, 680, 681, 682, 683, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>liongrove</th>\n",
       "      <td>30</td>\n",
       "      <td>{771, 772, 773, 646, 647, 648, 649, 650, 654, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sh</th>\n",
       "      <td>30</td>\n",
       "      <td>{771, 772, 773, 646, 647, 648, 649, 650, 654, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>culture</th>\n",
       "      <td>26</td>\n",
       "      <td>{652, 780, 666, 667, 668, 669, 670, 671, 672, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>arts</th>\n",
       "      <td>26</td>\n",
       "      <td>{652, 780, 666, 667, 668, 669, 670, 671, 672, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>centre</th>\n",
       "      <td>26</td>\n",
       "      <td>{652, 780, 666, 667, 668, 669, 670, 671, 672, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mercurytours</th>\n",
       "      <td>22</td>\n",
       "      <td>{1537, 1538, 1541, 1549, 1475, 1476, 1477, 148...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>harryandrowenaphotos</th>\n",
       "      <td>22</td>\n",
       "      <td>{1537, 1538, 1541, 1549, 1475, 1476, 1477, 148...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>street</th>\n",
       "      <td>21</td>\n",
       "      <td>{896, 641, 897, 900, 901, 2950, 395, 2446, 233...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>temple</th>\n",
       "      <td>21</td>\n",
       "      <td>{774, 775, 2441, 2442, 2443, 4111, 3484, 3628,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>jeans</th>\n",
       "      <td>20</td>\n",
       "      <td>{1551, 1552, 1553, 1554, 1555, 1556, 1557, 155...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pagoda</th>\n",
       "      <td>19</td>\n",
       "      <td>{2818, 774, 775, 2444, 3628, 2863, 2870, 2871,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>w</th>\n",
       "      <td>18</td>\n",
       "      <td>{3616, 3617, 3618, 3619, 3620, 3621, 3622, 395...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shanghai</th>\n",
       "      <td>18</td>\n",
       "      <td>{3680, 641, 2818, 3108, 3109, 3110, 2439, 2440...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               count  \\\n",
       "china                            672   \n",
       "Suzhou                           543   \n",
       "jiangsu                          293   \n",
       "peoplesrepublicofchina           217   \n",
       "garden                           173   \n",
       "unescoworldheritage              144   \n",
       "classicalgardens                 133   \n",
       "suzhoushi                         60   \n",
       "jiangsusheng                      60   \n",
       "places                            60   \n",
       "gardenofthehumbleadministrator    54   \n",
       "humble                            44   \n",
       "administrator                     41   \n",
       "cn                                33   \n",
       "zhu                               32   \n",
       "lingering                         31   \n",
       "at the                            31   \n",
       "liongrove                         30   \n",
       "sh                                30   \n",
       "culture                           26   \n",
       "arts                              26   \n",
       "centre                            26   \n",
       "mercurytours                      22   \n",
       "harryandrowenaphotos              22   \n",
       "street                            21   \n",
       "temple                            21   \n",
       "jeans                             20   \n",
       "pagoda                            19   \n",
       "w                                 18   \n",
       "shanghai                          18   \n",
       "\n",
       "                                                                           origin  \n",
       "china                           {4111, 4116, 4117, 2099, 2115, 2116, 2117, 211...  \n",
       "Suzhou                          {4111, 108, 2901, 349, 350, 351, 2439, 2442, 3...  \n",
       "jiangsu                         {3615, 3616, 3617, 3618, 3619, 3620, 3621, 550...  \n",
       "peoplesrepublicofchina          {3615, 3616, 3617, 3618, 3619, 3620, 3621, 550...  \n",
       "garden                          {3615, 3616, 4129, 3617, 3618, 3619, 3620, 362...  \n",
       "unescoworldheritage             {3615, 3616, 3617, 3618, 3619, 3620, 3621, 362...  \n",
       "classicalgardens                {3615, 3616, 3617, 3618, 3619, 3620, 3621, 362...  \n",
       "suzhoushi                       {768, 781, 784, 785, 658, 787, 660, 4116, 4117...  \n",
       "jiangsusheng                    {768, 781, 784, 785, 658, 787, 660, 4116, 4117...  \n",
       "places                          {3840, 3841, 3842, 3843, 3844, 3845, 3846, 384...  \n",
       "gardenofthehumbleadministrator  {3242, 3243, 3244, 3245, 3246, 3247, 3248, 324...  \n",
       "humble                          {768, 1908, 2819, 1909, 1910, 395, 781, 1911, ...  \n",
       "administrator                   {768, 1909, 1910, 781, 1911, 784, 785, 658, 78...  \n",
       "cn                              {4116, 4117, 3484, 3485, 3486, 3487, 3488, 348...  \n",
       "zhu                             {768, 781, 784, 785, 658, 787, 660, 788, 789, ...  \n",
       "lingering                       {653, 783, 677, 678, 679, 680, 681, 682, 683, ...  \n",
       "at the                          {653, 783, 677, 678, 679, 680, 681, 682, 683, ...  \n",
       "liongrove                       {771, 772, 773, 646, 647, 648, 649, 650, 654, ...  \n",
       "sh                              {771, 772, 773, 646, 647, 648, 649, 650, 654, ...  \n",
       "culture                         {652, 780, 666, 667, 668, 669, 670, 671, 672, ...  \n",
       "arts                            {652, 780, 666, 667, 668, 669, 670, 671, 672, ...  \n",
       "centre                          {652, 780, 666, 667, 668, 669, 670, 671, 672, ...  \n",
       "mercurytours                    {1537, 1538, 1541, 1549, 1475, 1476, 1477, 148...  \n",
       "harryandrowenaphotos            {1537, 1538, 1541, 1549, 1475, 1476, 1477, 148...  \n",
       "street                          {896, 641, 897, 900, 901, 2950, 395, 2446, 233...  \n",
       "temple                          {774, 775, 2441, 2442, 2443, 4111, 3484, 3628,...  \n",
       "jeans                           {1551, 1552, 1553, 1554, 1555, 1556, 1557, 155...  \n",
       "pagoda                          {2818, 774, 775, 2444, 3628, 2863, 2870, 2871,...  \n",
       "w                               {3616, 3617, 3618, 3619, 3620, 3621, 3622, 395...  \n",
       "shanghai                        {3680, 641, 2818, 3108, 3109, 3110, 2439, 2440...  "
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tag_df.head(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "id": "5b2711b5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>origin</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>mobile</th>\n",
       "      <td>1</td>\n",
       "      <td>{897}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>phone</th>\n",
       "      <td>1</td>\n",
       "      <td>{897}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>merchant</th>\n",
       "      <td>1</td>\n",
       "      <td>{900}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hp</th>\n",
       "      <td>1</td>\n",
       "      <td>{2740}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pushedfilm</th>\n",
       "      <td>1</td>\n",
       "      <td>{2740}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>geometry</th>\n",
       "      <td>1</td>\n",
       "      <td>{613}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>iphonex</th>\n",
       "      <td>1</td>\n",
       "      <td>{621}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>symmetry</th>\n",
       "      <td>1</td>\n",
       "      <td>{613}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Impe</th>\n",
       "      <td>1</td>\n",
       "      <td>{613}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>modernist</th>\n",
       "      <td>1</td>\n",
       "      <td>{613}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ceiling</th>\n",
       "      <td>1</td>\n",
       "      <td>{613}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>canglangpavilion</th>\n",
       "      <td>1</td>\n",
       "      <td>{4129}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pavilionofsurgingwaves</th>\n",
       "      <td>1</td>\n",
       "      <td>{4129}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mirror</th>\n",
       "      <td>1</td>\n",
       "      <td>{621}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mind</th>\n",
       "      <td>1</td>\n",
       "      <td>{621}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shouzuo</th>\n",
       "      <td>1</td>\n",
       "      <td>{621}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>streetphoto</th>\n",
       "      <td>1</td>\n",
       "      <td>{621}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>beauty</th>\n",
       "      <td>1</td>\n",
       "      <td>{641}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ilfordhp</th>\n",
       "      <td>1</td>\n",
       "      <td>{2740}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lady</th>\n",
       "      <td>1</td>\n",
       "      <td>{641}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>modern</th>\n",
       "      <td>1</td>\n",
       "      <td>{641}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>confident</th>\n",
       "      <td>1</td>\n",
       "      <td>{641}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>woman</th>\n",
       "      <td>1</td>\n",
       "      <td>{641}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>beautiful</th>\n",
       "      <td>1</td>\n",
       "      <td>{641}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mainland</th>\n",
       "      <td>1</td>\n",
       "      <td>{641}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>water reflection</th>\n",
       "      <td>1</td>\n",
       "      <td>{2739}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hasselblad</th>\n",
       "      <td>1</td>\n",
       "      <td>{2740}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mediumformat</th>\n",
       "      <td>1</td>\n",
       "      <td>{2740}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cinasuzhouautumn</th>\n",
       "      <td>1</td>\n",
       "      <td>{4140}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>empty</th>\n",
       "      <td>1</td>\n",
       "      <td>{4013}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       count  origin\n",
       "mobile                     1   {897}\n",
       "phone                      1   {897}\n",
       "merchant                   1   {900}\n",
       "hp                         1  {2740}\n",
       "pushedfilm                 1  {2740}\n",
       "geometry                   1   {613}\n",
       "iphonex                    1   {621}\n",
       "symmetry                   1   {613}\n",
       "Impe                       1   {613}\n",
       "modernist                  1   {613}\n",
       "ceiling                    1   {613}\n",
       "canglangpavilion           1  {4129}\n",
       "pavilionofsurgingwaves     1  {4129}\n",
       "mirror                     1   {621}\n",
       "mind                       1   {621}\n",
       "shouzuo                    1   {621}\n",
       "streetphoto                1   {621}\n",
       "beauty                     1   {641}\n",
       "ilfordhp                   1  {2740}\n",
       "lady                       1   {641}\n",
       "modern                     1   {641}\n",
       "confident                  1   {641}\n",
       "woman                      1   {641}\n",
       "beautiful                  1   {641}\n",
       "mainland                   1   {641}\n",
       "water reflection           1  {2739}\n",
       "hasselblad                 1  {2740}\n",
       "mediumformat               1  {2740}\n",
       "cinasuzhouautumn           1  {4140}\n",
       "empty                      1  {4013}"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tag_df.tail(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "id": "eb254223",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAdQUlEQVR4nO3deXRc5Z3m8e9PpdJia7FKkm0sL1V2DI4xHS9Smd0QSAfS3QGy4iFLBybG3aGTyWSmD0z3SXM6nZ50MgknzEkAM9DASZo1ZOJMSBNCJxgCxJZtvGHAuy1vkixrsyS7pHrnD5WxMJK1VJVu1a3nc04dVb333ro/+T166vqte99rzjlERMRf8rwuQEREUk/hLiLiQwp3EREfUriLiPiQwl1ExIcU7iIiPpTvdQEAVVVVLhwOe12GiEhWWb9+fbNzrnqwZRkR7uFwmPr6eq/LEBHJKma2b6hlGpYREfEhhbuIiA8p3EVEfEjhLiLiQwp3EREfUriLiPhQVod7R0+Mf996mNauU16XIiKSUbI63Hc2drLyJxt4ddcxr0sREckoWR3uC2rKKQ4GWLunxetSREQySlaHezCQx+JZkxTuIiJnyepwB6gLh9h+pJ32npjXpYiIZIysD/doOIRzsH7vca9LERHJGFkf7otmVpCfZ6zdq6EZEZHTsj7ciwsCXDS9XOPuIiIDZH24A0QjITY3tNIT6/O6FBGRjOCPcA+HiPU53jjQ6nUpIiIZwRfhXjsrhBkamhERSfBFuJdPCHLBlFLW6UtVERHAJ+EOsDQSYv2+4/T2xb0uRUTEc74J97pIiK5TfWw71O51KSIinvNNuEfDIUDj7iIi4KNwn1xWRLhygi5mEhFhBOFuZg+bWaOZbR3QdreZHTSzNxKPjw1YdpeZ7TSzt83so+kqfDB14RDr9rYQj7vx3K2ISMYZyZH7I8B1g7Tf45xbmHg8B2Bm84GbgQsT2/zYzAKpKnY40UiI1q4YO5s6x2uXIiIZadhwd86tAUY61nED8IRz7qRzbg+wE4gmUd+oRCMadxcRgeTG3O8ws82JYZuKRFsNcGDAOg2JtnExMzSBKWWFCncRyXljDff7gDnAQuAw8P3RvoGZrTCzejOrb2pqGmMZ73tP6sIh1u5pwTmNu4tI7hpTuDvnjjrn+pxzceBBzgy9HARmDFh1eqJtsPdY5Zyrdc7VVldXj6WMQS2NhDjS3kPD8e6UvaeISLYZU7ib2XkDXt4EnD6TZjVws5kVmlkEmAusTa7E0anTuLuICPnDrWBmjwNXAVVm1gD8A3CVmS0EHLAXuB3AObfNzJ4C3gR6ga8458Z1Ht7zJ5dSXhxk7Z4WPrlk+njuWkQkYwwb7s655YM0P3SO9b8NfDuZopKRl2fUhSs0iZiI5DTfXKE6UF04xO7mEzR29HhdioiIJ3wZ7qfPd6/XTbNFJEf5MtwX1JRTHAzoS1URyVm+DPdgII/FsyYp3EUkZ/ky3KF/3H37kXbae2JelyIiMu58G+7RSAjnYL3G3UUkB/k23BfNqCAYMM3vLiI5ybfhXlwQ4KKaco27i0hO8m24Q/9UBJsbWumJjetFsiIinvN1uEfDIWJ9jo37W70uRURkXPk63GtnhTBDUxGISM7xdbiXTwhywZRSjbuLSM7xdbhD//zuG/YfJ9YX97oUEZFx4/twr4uE6DrVx7ZD7V6XIiIybnwf7tFw/yRi6zQ0IyI5xPfhPrmsiHDlBF3MJCI5xffhDv1TEazb20I8rptmi0huyIlwrwuHaO2KsbOp0+tSRETGRU6E+9JIJQB/1Li7iOSInAj3GaFippQV6ktVEckZORHuZkZdOMTaPS04p3F3EfG/YcPdzB42s0Yz2zqg7Xtm9paZbTazn5vZpER72My6zeyNxOP+NNY+KksjIY6099BwvNvrUkRE0m4kR+6PANed1fYCsMA59yfAO8BdA5btcs4tTDxWpqbM5NUlbpqtcXcRyQXDhrtzbg3Qclbbb5xzvYmXrwPT01BbSp0/uZTy4qDG3UUkJ6RizP1W4NcDXkfMbKOZvWRmV6Tg/VMiL8+oC1dohkgRyQlJhbuZ/R3QC/w00XQYmOmcWwT8V+DfzKxsiG1XmFm9mdU3NTUlU8aIRSMhdjefoLGjZ1z2JyLilTGHu5n9JfDnwC0ucQqKc+6kc+5Y4vl6YBdw/mDbO+dWOedqnXO11dXVYy1jVOoS88zU66bZIuJzYwp3M7sO+Fvg4865rgHt1WYWSDyfDcwFdqei0FRYUFNOcTCg+d1FxPfyh1vBzB4HrgKqzKwB+Af6z44pBF4wM4DXE2fGXAn8o5nFgDiw0jmXMUkaDOSxeNYkhbuI+N6w4e6cWz5I80NDrPsz4GfJFpVOdeEQP3xxB23dMcqLg16XIyKSFjlxhepA0UgI52DDPo27i4h/5Vy4L5pRQTBguphJRHwt58K9uCDARTXlOt9dRHwt58Id+qci2NzQSk+sz+tSRETSIifDfWkkRKzPsXF/q9eliIikRU6G+5JZIczQ0IyI+FZOhnt5cZB5U8t0vruI+FZOhjtANFzBhv3HifXFvS5FRCTlcjbc6yIhuk71se1Qu9eliIikXM6GezQxiZjmdxcRP8rZcJ9cVkS4coIuZhIRX8rZcIf+qQjq97UQj+um2SLiLzkd7nXhEK1dMXY0dnpdiohISuV0uC+NVAKwVue7i4jP5HS4zwgVM6WsUOe7i4jv5HS4mxnRSCXr9rSQuFOgiIgv5HS4Q//FTEfae2g43u11KSIiKaNwT4y765RIEfGTnA/3uZNLKC8O6mImEfGVnA/3vDyjLlyhM2ZExFdyPtyh/2KmPc0naOzo8boUEZGUGFG4m9nDZtZoZlsHtIXM7AUz25H4WZFoNzO718x2mtlmM1ucruJTpe7deWZ002wR8YeRHrk/Alx3VtudwIvOubnAi4nXANcDcxOPFcB9yZeZXgtqyikOBnTzDhHxjRGFu3NuDXB28t0APJp4/ihw44D2x1y/14FJZnZeCmpNm2Agj8WzJumMGRHxjWTG3Kc45w4nnh8BpiSe1wAHBqzXkGjLaNFwJW8daaetO+Z1KSIiSUvJF6qu//LOUV3iaWYrzKzezOqbmppSUUZS6iIVOAcb9mncXUSyXzLhfvT0cEviZ2Oi/SAwY8B60xNt7+GcW+Wcq3XO1VZXVydRRmosmlFBMGAamhERX0gm3FcDX0w8/yLwiwHtX0icNXMx0DZg+CZjFRcEuKimXF+qiogvjPRUyMeB14ALzKzBzG4DvgN8xMx2ANcmXgM8B+wGdgIPAn+d8qrTpC4SYnNDKz2xPq9LERFJSv5IVnLOLR9i0TWDrOuAryRTlFeWRkI88NJuNu5v5ZI5lV6XIyIyZrpCdYAls0KYofndRSTrKdwHKC8OMm9qmcbdRSTrKdzPEg1XsH7fcWJ9ca9LEREZM4X7WaKRSrpjfWw71O51KSIiY6ZwP0tdpAKAtXuOeVyJiMjYKdzPMrm0iEjVRNZqhkgRyWIK90HUhSuo39dCPK6bZotIdlK4DyIaqaS1K8aOxk6vSxERGROF+yCiiZt36NZ7IpKtFO6DmBEqZkpZoS5mEpGspXAfhJkRjVSybk8L/bMpiIhkF4X7EKLhCo6093CgpdvrUkRERk3hPoRopH/iMI27i0g2UrgPYe7kEsqLg7qYSUSyksJ9CHl5Rl04xLq9uphJRLKPwv0copEK9jSfoLGjx+tSRERGReF+DqfH3ddpKgIRyTIK93O4cFoZxcGA5ncXkayjcD+HYCCPxbMm8UddzCQiWUbhPoxouJK3jrTT1h3zuhQRkRFTuA+jLlKBc7B+n47eRSR7KNyHsWhGBcGAaX53Eckq+WPd0MwuAJ4c0DQb+CYwCfgy0JRo/x/OuefGuh+vFRcEuKimXBcziUhWGfORu3PubefcQufcQmAJ0AX8PLH4ntPLsjnYT4tGKtlysI3uU31elyIiMiKpGpa5BtjlnNuXovfLKNFIBbE+x8YDGpoRkeyQqnC/GXh8wOs7zGyzmT1sZhWDbWBmK8ys3szqm5qaBlslYyyZFcJMFzOJSPZIOtzNrAD4OPB0ouk+YA6wEDgMfH+w7Zxzq5xztc652urq6mTLSKvy4iDzppbpYiYRyRqpOHK/HtjgnDsK4Jw76pzrc87FgQeBaAr24blouIL1+44T64t7XYqIyLBSEe7LGTAkY2bnDVh2E7A1BfvwXDRSSXesj22H2r0uRURkWEmFu5lNBD4CPDug+btmtsXMNgNXA19PZh+Zoi7S/9WBTokUkWww5vPcAZxzJ4DKs9o+n1RFGWpyaRGRqoms3XOcFVd6XY2IyLnpCtVRqAtXsG5vC/G4bpotIplN4T4K0Uglbd0xdjR2el2KiMg5KdxHIRoOARp3F5HMp3AfhRmhYqaWFbFW91UVkQyncB8FM6MuEmLtnmM4p3F3EclcCvdRikZCHG0/yYGWbq9LEREZksJ9lN4dd9dUBCKSwRTuozR3cgnlxUF9qSoiGU3hPkp5eUZdOMQ6fakqIhlM4T4G0UgFe5pP0NjR43UpIiKDUriPQTTSP+OC5ncXkUylcB+DC6eVURwMaNxdRDKWwn0MgoE8lsyq0MVMIpKxFO5jVBcO8daRdtq6Y16XIiLyPgr3MYpGQjgH6/fpfHcRyTwK9zFaNHMSwYCxVl+qikgGUriPUVEwwEU15fpSVUQyksI9CdFIJVsOttF9qs/rUkRE3kPhnoRopIJYn2PjAQ3NiEhmUbgnYcmsEGa6mElEMo/CPQnlxUEWTCvn+W1HNL+7iGSUpMPdzPaa2RYze8PM6hNtITN7wcx2JH5WJF9qZvrcxTN583A7f9ipL1ZFJHOk6sj9aufcQudcbeL1ncCLzrm5wIuJ175046IaqksLeWDNLq9LERF5V7qGZW4AHk08fxS4MU378VxhfoBbL4vw8o5mth5s87ocEREgNeHugN+Y2XozW5Fom+KcO5x4fgSYcvZGZrbCzOrNrL6pqSkFZXjnlotnUlKYzwNrdntdiogIkJpwv9w5txi4HviKmV05cKHr/6bxfd82OudWOedqnXO11dXVKSjDO2VFQW5ZOpNfbT7EgZYur8sREUk+3J1zBxM/G4GfA1HgqJmdB5D42ZjsfjLdrZdHCOQZD76so3cR8V5S4W5mE82s9PRz4E+BrcBq4IuJ1b4I/CKZ/WSDKWVF3LSohqfqD3Cs86TX5YhIjkv2yH0K8IqZbQLWAr9yzv078B3gI2a2A7g28dr3Vlw5m55YnEdf2+d1KSKS4/KT2dg5txv40CDtx4BrknnvbPSByaV8ZP4UHnttLyuXzWZCQVL/vCIiY6YrVFNs5bLZtHbFeGrdAa9LEZEcpnBPsSWzQtSFK3jw5T3E+uJelyMiOUrhnga3XzmHg63dPLfl8PAri4ikgcI9DT48bzJzJ5dw/0u7NaGYiHhC4Z4GeXnGiitns/1wO2t2NHtdjojkIIV7mtywsIapZUU88JImFBOR8adwT5OC/DxuuzzCq7uOsbmh1etyRCTHKNzT6OboDEqL8nngJU1JICLjS+GeRqVFQT538Sx+vfUwe5tPeF2OiOQQhXuafemyMPl5eZpQTETGlcI9zSaXFvHJJTU8vb6Bpg5NKCYi40PhPg6+fMVsYn1xHn11r9eliEiOULiPg9nVJXx0/lQee20vJ072el2OiOQAhfs4uX3ZbNp7enl87X6vSxGRHKBwHyeLZlawNBLioVc0oZiIpJ/CfRytXDaHw209rH7jkNeliIjPKdzH0VUXVHPBlFIeWLNLE4qJSFop3MeRmXH7stm8c7ST37/d5HU5IuJjCvdx9hcfmsa08iLu04RiIpJGCvdxFgzkcdsVs1m7p4UN+497XY6I+JTC3QM3182gvDio6YBFJG3GHO5mNsPMfmdmb5rZNjP7WqL9bjM7aGZvJB4fS125/jCxMJ8vXDKL37x5lF1NnV6XIyI+lMyRey/wDefcfOBi4CtmNj+x7B7n3MLE47mkq/ShL14aJhjI48E1mlBMRFJvzOHunDvsnNuQeN4BbAdqUlWY31WVFPLpJdN5dsNBGtt7vC5HRHwmJWPuZhYGFgF/TDTdYWabzexhM6sYYpsVZlZvZvVNTbl5WuCXr5hNbzzOv2pCMRFJsaTD3cxKgJ8B/8U51w7cB8wBFgKHge8Ptp1zbpVzrtY5V1tdXZ1sGVkpXDWR6xecx09e30dHT8zrckTER5IKdzML0h/sP3XOPQvgnDvqnOtzzsWBB4Fo8mX61+3LZtOhCcVEJMWSOVvGgIeA7c65HwxoP2/AajcBW8denv/9yfRJXDqnkode2cPJ3j6vyxERn0jmyP0y4PPAh8867fG7ZrbFzDYDVwNfT0Whfnb7sjkcbT/JLzShmIikSP5YN3TOvQLYIIt06uMoXTm3ig+eV8aqNbv51OLp5OUN9s8Ksb44xzpP0dx5kqbOkzR3nKS58xQlhQH+09JZBIbYTkRyz5jDXVLHzFi5bDZfe+INvvv825QUBmjuPDUgwPtDvK176C9d27pj3PHhueNYtYhkMoV7hvizi87jnhfe4f7ElASlhflUlRZSVVLA+VNKuXROIVUlhVSVFvT/LCmkOvH6rme3cM9vd3DJnCqWzBr0zFMRyTGWCfOK19bWuvr6eq/L8Fxbd4yOnhhVJYUUBQMj3q69J8bHfvgyAM997QrKioLpKlFEMoiZrXfO1Q62TBOHZZDy4iDTKyaMKtgByoqC3Lt8EYfbevj7n2/VjUBEROHuF4tnVvD1a+eyetMhnt1w0OtyRMRjCncf+aurPsDSSIhv/mIre5tPeF2OiHhI4e4jgTzjns8uJD+Qx9ee2Mip3rjXJYmIRxTuPjNtUjH/8smL2NTQxg9eeMfrckTEIwp3H7puwXksj87kgTW7+MPOZq/LEREPKNx96pt/Pp851SV8/ck3aDlxyutyRGScKdx9qrggwL03L6K1K8bfPrNJp0eK5BiFu4/Nn1bGndfP47fbG/nJ6/u8LkdExpHC3ee+dFmYqy+o5p9+tZ23j3R4XY6IjBOFu8+ZGd/79IcoLQry1cc30hMbvznj++IaChLxisI9B1SVFPKDz3yIt492cMe/beDEyd607s85x92rt3Hpd16kqeNkWvclIoNTuOeIK8+v5ls3LuA/3mrk0/e/xuG27rTt657f7uCRV/dytP0k33v+rRFvt/9YF8tXvc4TuuWgSNIU7jnk8xfP4qG/rGN/Sxc3/ugPbD3YlvJ9PPrqXu59cQefqZ3Ol6+I8PT6BjYdaB12uy0NbXzivj/w+p5j3PnsFh56ZU/KaxPJJQr3HHP1BZN55q8uIT8vj0/f/xrPbzuSsvdevekQd/9yG9d+cAr/fNNFfPWauVROLOTuX24jfo7x99+/3chnV71GUTDAr792BdcvmMq3/t+b/Oh3O8+5v3jcsaupk9WbDvG7txtT9nuI+IHmc89RjR09rHhsPZsaWrnzunmsuHI2/fc8H5uXdzRx6yPrWDSjgsdui747bfHT9Qf4789s5gef+RCfWDz9fds9XX+AO5/dwryppfzrl+qYXFpEb1+cbzy9iV+8cYiVy+aw7PxqAByOhuPdvHmona0H29h+uJ0Tp858QXzrZRH+7s8+qNsNSs4413zuCvcc1hPr4xtPb+JXmw9zc90MvnXjAoKB0f9nbtOBVpY/+DozQxN48vZLKC8+c7OQeNxx032vcqi1m9/9t6soKey/+Vdf3PHd59/igZd2c8XcKu773JJ3l51eftezm3mqvuF9+5tQEGD+eWVcOK2MC6eVM39aGc+sb+CRV/dyzbzJ3Lt8ERMLU3OTsd6+OIE8S+qDL13icUfcOfLH0GfiDwp3GVI87rjnt+/wv/9jJ5fOqeS+W5ZQPiFI16lemjsS93FNPE7fnLu58yTNHWdu1N3R08uMUDE/W3kpk8uK3rePjfuPc9OPX2Xlsjncef082rpifPWJjbz0ThOfv3gW3/yL+YN+qMTjjs0H2+gecHReXVpIpGrioEfnj722l7tXb2NqWRG3Xh7hs3UzKB3hXamOnzjFrqZOdjedYFdTJ7uaTrC7qZP9LV0UBwPMO6+UeVPL3v15wdTS93wYjaeeWB9PrjvAAy/t4lBbDxMLApQXByk7/SgKUl4cTLTl9/8sCr67zsBlxcFARn5wych4Eu5mdh3wQyAA/B/n3HeGWlfh7r2frW/gzmc3UxQM0Bd3dJ0a/Hz48uIgVSWJ+7iW9t/Htbq0kJsW1TBtUvGQ7/+Npzbxy02H+PEti/n2c9tpON7FP96wgOXRmSn9PV7ffYwfvPAOa/e0UFqYz+cumcWtl0WoLi18d52N+49z74s72N/ShaM/2I93nbn5eEEgj3DVBOZUlxCpmkjnyV7eOtzB9iPtdPScOY20ZlIxRcE8LpxWzrduXPCe/7GkQ/epPn76x32sWrObxo6T1M6q4PK5VbR399LeE6Otu//RfvrR00vnMKe9BgP2nuA/Hf5lRfkDPgSC7/mAOP3BUFoU1BCYx8Y93M0sALwDfARoANYBy51zbw62vsI9M9TvbeHJdQcoKw4mbsJ95mbcVaUFVE4spCB/bEMAje09XP2/fs+JU31UlRRw/+eWUBsOpfg3OGPTgVZWvbyb57YcpiCQxzUfnExRfoCmzpO8vKOZyokFXDy7EgzKivKZU13CnOoSZldPZHrFhEFDyznHwdZu3jrcwVtH2tnZ2ElPLM5vtx+lpqKYJTPTd3PyuHO8srOZ5s5TXDK7kr+55gNcMrty2KPu3r44HT29Z4K/5/QHwGBtZz4UTi8b7kK00sL8AR8K/R8IEwryUeSP3MKZk/jCJeExbetFuF8C3O2c+2ji9V0Azrn/Odj6Cvfc8HT9AX615TD/fNNF5zzKT6XdTZ088NJuXtt9DIcjmJfHjYtquPXySMqGVdbuaeHu1dvoOBkbfuUkfKC6hL+++gPUpfFDcSDn+v8H9+4HQNd7g799wIdD+4APjK5Yei+S85tr5k3h7o9fOKZtvQj3TwHXOef+c+L154Glzrk7BqyzAlgBMHPmzCX79mliKxGR0ThXuHv2NbtzbpVzrtY5V1tdXe1VGSIivpSucD8IzBjwenqiTURExkG6wn0dMNfMImZWANwMrE7TvkRE5CxpOVHXOddrZncAz9N/KuTDzrlt6diXiIi8X9quwnDOPQc8l673FxGRoem6ZRERH1K4i4j4kMJdRMSHMmLiMDNrAs6+iqkcGOxuEoO1VwHNaShtpIaqdbzeazTbDLfuWJcP1p6JfQXZ018jWe9c64ymr4Zq97q/vO6r0WznRX9Ncs4NfqGQcy4jH8CqkbYD9ZlY63i912i2GW7dsS4fol8yrq+yqb9Gst651hlNX2Vqf3ndV9nWXwMfmTws88tRtnsplTWN5b1Gs81w6451+WDtmdhXkD39NZL1zrXOaP+GMrG/vO6r0WyXUf2VEcMyyTKzejfE/AqSWdRX2UX9lb0y+ch9NFZ5XYCMmPoqu6i/spQvjtxFROS9/HLkLiIiAyjcRUR8SOEuIuJDvgt3M5toZo+a2YNmdovX9ci5mdlsM3vIzJ7xuhYZnpndmPjbetLM/tTremRoWRHuZvawmTWa2daz2q8zs7fNbKeZ3Zlo/gTwjHPuy8DHx71YGVV/Oed2O+du86ZSgVH31/9N/G2tBD7rRb0yMlkR7sAjwHUDG8wsAPwIuB6YDyw3s/n03/XpQGK1vnGsUc54hJH3l3jvEUbfX3+fWC4ZKivC3Tm3Bmg5qzkK7Ewc+Z0CngBuABroD3jIkt/Pb0bZX+Kx0fSX9fsX4NfOuQ3jXauMXDaHXw1njtChP9RrgGeBT5rZfWTm5dS5atD+MrNKM7sfWGRmd3lTmgxiqL+vvwGuBT5lZiu9KExGJm13YvKKc+4E8CWv65CRcc4do3/8VrKAc+5e4F6v65DhZfOR+0FgxoDX0xNtkpnUX9lF/ZXlsjnc1wFzzSxiZgXAzcBqj2uSoam/sov6K8tlRbib2ePAa8AFZtZgZrc553qBO4Dnge3AU865bV7WKf3UX9lF/eVPmjhMRMSHsuLIXURERkfhLiLiQwp3EREfUriLiPiQwl1ExIcU7iIiPqRwFxHxIYW7iIgPKdxFRHzo/wMx0fwD043ylQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "tag_df['count'].value_counts().sort_index().plot(logx=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b2a99c0b",
   "metadata": {},
   "source": [
    "#### Deciding on the dimension of the embedding"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14916492",
   "metadata": {},
   "source": [
    "The number of 2 seems to provide an elbow point for the remained tags for embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "id": "5a0357f5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zzbn1\\AppData\\Local\\Temp\\ipykernel_26468\\461701484.py:3: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  xs.append(len(tag_df[tag_df['count']>i][tag_df['count']<200]))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAeGklEQVR4nO3deXRc5Znn8e+jXbI226qSjSxbXmQZQ7ANxmCTgARkoMkCSXcSyGIDSZPTIZ1kkpmEcM4M3SfDOaQnCZNMT5gmgY5JWIdAwxBOGiaRQyBgWzZm8b7gTRhJXmRL1i4980dd2zLIlqzFt5bf5xyduvXeW6VHlfhXL289t665OyIiklzSwi5ARERGn8JdRCQJKdxFRJKQwl1EJAkp3EVEklBG2AUAlJSUeEVFRdhliIgklDVr1ux398hA++Ii3CsqKqirqwu7DBGRhGJmu061T8syIiJJSOEuIpKEFO4iIklI4S4ikoQU7iIiSUjhLiKShBTuIiJJKKHDfUtDCz94bgOdPb1hlyIiElcSOtz3HmrjgZffYdU7B8MuRUQkriR0uC+eUUJWRhq1m5rCLkVEJK4kdLjnZqWzeMZEVmxuDLsUEZG4ktDhDlBTFWHH/qPsOnA07FJEROJGwod7dVUUgBWbtTQjInJMwod7Rck4ppeMo1ZLMyIixyV8uANUV0V4dfsB2rvUEikiAkkS7jVVUTp7+nhtx4GwSxERiQtJEe6Lpk8gNzNdXTMiIoGkCPeczHSWzJxI7eYm3D3sckREQpcU4Q5QPSfK7oNt7NivlkgRkeQJ99mxa8TWbtLSjIhI0oR7+YQ8KqP5/GmL+t1FRJIm3AFq5kRZueMgRzt7wi5FRCRUSRXu1bMjdPX28ZftaokUkdSWVOG+sGIC47LSdbaqiKS8pAr3rIw0PlxZwopNjWqJFJGUllThDrGzVd893MHWxtawSxERCU3ShfsVVWqJFBFJunCfXJTLnEkFWncXkZQ2aLibWY6ZrTKzN8xsvZn9YzA+3cxWmtk2M3vczLKC8ezg/rZgf8UY/w0fUDMnSt3OQxzp6D7bv1pEJC4MZebeCVzp7vOA+cC1ZnYp8EPgXnefBRwCvhwc/2XgUDB+b3DcWVVTFaWnz3ll6/6z/atFROLCoOHuMcc+ncwMfhy4EngyGF8O3BBsXx/cJ9h/lZnZaBU8FBdOLaYgJ0NXZxKRlDWkNXczSzezdUAj8CKwHWh292Ongu4FyoLtMmAPQLD/MDBxgOe8zczqzKyuqWl0QzgjPY3LKyPUblZLpIikpiGFu7v3uvt8YAqwCJgz0l/s7ve7+0J3XxiJREb6dB9QXRWhsaWTDfuOjPpzi4jEuzPqlnH3ZqAWWAwUm1lGsGsKUB9s1wPlAMH+IuCsfx/AsZZILc2ISCoaSrdMxMyKg+1c4KPARmIh/zfBYcuAZ4LtZ4P7BPv/6CGsjUQLcvhQWZH63UUkJQ1l5j4ZqDWzN4HVwIvu/hzwPeDbZraN2Jr6A8HxDwATg/FvA3eMftlDU1MVYe3uQxxuU0ukiKSWjMEOcPc3gQUDjO8gtv7+/vEO4DOjUt0IXVEV5Wd/3MZLW5v4xLxzwi5HROSsSbozVPubX15McV6mzlYVkZST1OGenmZcMTvCnzY30denlkgRSR1JHe4QO1v1wNEu3qo/HHYpIiJnTdKH++WzI5ipJVJEUkvSh/uEcVnMm1KsdXcRSSlJH+4QW5p5Y28zB1o7wy5FROSsSI1wnxPBHV7aqqUZEUkNKRHu559TREl+FrWbFO4ikhpSItzT0owrZkd5aWsTvWqJFJEUkBLhDrFviWxu62bdnuawSxERGXMpE+6XV0ZIM1ihrhkRSQEpE+5FeZlcNG28WiJFJCWkTLgDVFdFebv+CI0tHWGXIiIyplIq3GuqogD8SWerikiSS6lwP3dyAaWF2foqAhFJeikV7mZGddAS2d3bF3Y5IiJjJqXCHWJnq7Z09LB216GwSxERGTMpF+6XzSohI82o1dKMiCSxlAv3gpxMLq6YoH53EUlqKRfuEDtbddN7Lew73B52KSIiYyIlw71mTqwlUl0zIpKsUjLcK6P5lBXnUrtJSzMikpxSMtzNjOqqCK9s209Xj1oiRST5pGS4Q+xs1aNdvdTtPBh2KSIioy5lw33JrIlkpafpi8REJCmlbLjnZWVwyYwJ6ncXkaQ0aLibWbmZ1ZrZBjNbb2bfDMb/wczqzWxd8HNdv8d838y2mdlmM7tmLP+AkaiuirKtsZU9B9vCLkVEZFQNZebeA3zH3ecClwK3m9ncYN+97j4/+HkeINh3I3AecC3wczNLH4PaR6ymKgLoAh4iknwGDXd33+fua4PtFmAjUHaah1wPPObune7+DrANWDQaxY626SXjmDYxT/3uIpJ0zmjN3cwqgAXAymDo62b2ppk9aGbjg7EyYE+/h+3l9G8GoYl9S2SEV7bvp6O7N+xyRERGzZDD3czygd8C33L3I8B9wExgPrAP+PGZ/GIzu83M6sysrqkpvJlz9ZwoHd19rHxHLZEikjyGFO5mlkks2B9296cA3L3B3XvdvQ/4BSeWXuqB8n4PnxKMncTd73f3he6+MBKJjORvGJHFMyaSnZGms1VFJKkMpVvGgAeAje7+k37jk/sd9ing7WD7WeBGM8s2s+lAJbBq9EoeXTmZ6SyZOVEfqopIUskYwjGXAV8C3jKzdcHYncBNZjYfcGAn8FUAd19vZk8AG4h12tzu7nG9oF1dFaV283re2X+U6SXjwi5HRGTEBg13d38ZsAF2PX+ax9wN3D2Cus6qmqood7GeFZsbmV4yPexyRERGLGXPUO1v6sQ8ZkTG6WxVEUkaCvdATVWU13YcoL0rrleQRESGROEeqKmK0tXTx6s79oddiojIiCncAxdPH09eVjq1m7Q0IyKJT+EeyM5IZ8nMEmo3N+LuYZcjIjIiCvd+auZE2Huone1NrWGXIiIyIgr3fqqrYhfO1tKMiCQ6hXs/ZcW5VJUWsGKLzlYVkcSmcH+f6qoIq945SGtnT9iliIgMm8L9faqronT3Oq9sU0ukiCQuhfv7LKwYT352hr5ITEQSmsL9fTLT0/hIZQm1m5rUEikiCUvhPoCaqijvHelgc0NL2KWIiAyLwn0AVwQXzlZLpIgkKoX7AEoLc5g7uZBarbuLSIJSuJ9CzZwIa3Yd4nB7d9iliIicMYX7KdRURentc17eqpZIEUk8CvdTmF9eTFFuploiRSQhKdxPISNoiVyxpYm+PrVEikhiUbifRk1VlKaWTjbsOxJ2KSIiZ0ThfhonWiK1NCMiiUXhfhol+dnMm1KklkgRSTgK90FUV0VZt6eZQ0e7wi5FRGTIFO6DqK6K0Ofw0ladrSoiiUPhPogLphQzYVwWKzYr3EUkcSjcB5GeZlwxO8KftjTRq5ZIEUkQCvchqK6KcPBoF2/ubQ67FBGRIRk03M2s3MxqzWyDma03s28G4xPM7EUz2xrcjg/Gzcx+ZmbbzOxNM7twrP+IsXZ5ZYQ0Q0szIpIwhjJz7wG+4+5zgUuB281sLnAH8Ad3rwT+ENwH+CugMvi5Dbhv1Ks+y8aPy2J+ebG+ikBEEsag4e7u+9x9bbDdAmwEyoDrgeXBYcuBG4Lt64GHPOY1oNjMJo924WdbTVWUN/YepqmlM+xSREQGdUZr7mZWASwAVgKl7r4v2PUeUBpslwF7+j1sbzD2/ue6zczqzKyuqSn+lztq5kQBeGlL/NcqIjLkcDezfOC3wLfc/aQvW/HYxUbPqJXE3e9394XuvjASiZzJQ0Mxd3IhkYJsna0qIglhSOFuZpnEgv1hd38qGG44ttwS3B5LvXqgvN/DpwRjCS0tzaieHeHPW/fT09sXdjkiIqc1lG4ZAx4ANrr7T/rtehZYFmwvA57pN7406Jq5FDjcb/kmoVVXRTnc3s26Pc1hlyIiclpDmblfBnwJuNLM1gU/1wH3AB81s63A1cF9gOeBHcA24BfA10a/7HB8uLKE9DTT0oyIxL2MwQ5w95cBO8XuqwY43oHbR1hXXCrKzeSiaeOp3dTEf75mTtjliIicks5QPUM1VVE27DtCw5GOsEsRETklhfsZqg4u4PEnna0qInFM4X6G5kwqYFJhjtbdRSSuKdzPkJlRMyfWEtmtlkgRiVMK92GororS2tlD3c5DYZciIjIghfswXDarhMx00xeJiUjcUrgPQ352BhdXTNBXAItI3FK4D1NNVZTNDS3UN7eHXYqIyAco3IepZk6sJVJLMyISjxTuwzQzks+U8bnUbtLSjIjEH4X7MJkZNVVR/rJ9P509vWGXIyJyEoX7CFRXRWjr6mX1O2qJFJH4onAfgcUzJ5KVkaazVUUk7ijcRyAvK4NLZ0xUuItI3FG4j1BNVYQdTUfZfaAt7FJERI5TuI9QTVXswtkrtmj2LiLxQ+E+QhUl46iYmEftJoW7iMQPhfsouHJOKa9sO8CaXeqaEZH4oHAfBbfXzKRsfC5fXr6abY2tYZcjIqJwHw0T87NZfssiMtLSWPbgKl2CT0RCp3AfJVMn5vGrWy6mua2LZQ+u4nB7d9gliUgKU7iPovPLivjfX7qI7U2tfPXXdfpaAhEJjcJ9lH2kMsKPPjOP13Yc5NuPv0Ffn4ddkoikoIywC0hG188vo/FIJ3c/v5FIQTZ3fWIuZhZ2WSKSQhTuY+RvL59Bw5EOfvnyO5QW5vB31TPDLklEUojCfQzded25NLZ08sPfbyJakM1fXzQl7JJEJEUMuuZuZg+aWaOZvd1v7B/MrN7M1gU/1/Xb930z22Zmm83smrEqPBGkpRk/+sw8PjyrhO/99k1dtUlEzpqhfKD6K+DaAcbvdff5wc/zAGY2F7gROC94zM/NLH20ik1EWRlp3PfFC5ldWsDXHl7LG3uawy5JRFLAoOHu7i8BB4f4fNcDj7l7p7u/A2wDFo2gvqRQkJPJr269mIn5Wdz6q9Xs3H807JJEJMmNpBXy62b2ZrBsMz4YKwP29DtmbzCW8qIFOSy/ZREOLH1wFU0tnWGXJCJJbLjhfh8wE5gP7AN+fKZPYGa3mVmdmdU1NaXGRaZnRPJ58OaLaWrp5JZfraK1syfskkQkSQ0r3N29wd173b0P+AUnll7qgfJ+h04JxgZ6jvvdfaG7L4xEIsMpIyHNLy/m51+4kI37Wvi736yhq6cv7JJEJAkNK9zNbHK/u58CjnXSPAvcaGbZZjYdqARWjazE5FMzJ8o9n/4Qf966n+8+qbNYRWT0DdrnbmaPAtVAiZntBe4Cqs1sPuDATuCrAO6+3syeADYAPcDt7q4vWBnAZxaW09jSyX//982UFubw/evODbskEUkig4a7u980wPADpzn+buDukRSVKr5WPZOGIx38y0s7iBRk85WPzAi7JBFJEjpDNURmxl2fOI+mlk7+2+82Ei3M4ZPzzgm7LBFJAvpWyJClpxn3fm4+i6ZP4DtPrOOVbfvDLklEkoDCPQ7kZKbzi6ULmVGSz1d/vYb17x4OuyQRSXAK9zhRlBs7i7UwJ4Ob/3U1ew62hV2SiCQwhXscmVyUy/JbF9HV08fSB1dxoFVnsYrI8Cjc40xlaQEPLFvIu83t3Lq8jrYuncUqImdO4R6HFlZM4Gc3LeCtvc3c/vBaunt1FquInBmFe5y65rxJ/OCG86nd3MSdT72Fu85iFZGhU597HPvCJdNoPNLJT/+wldLCHP7TNVVhlyQiCULhHue+dXUljS0d/HPtNkoLs/nS4oqwSxKRBKBwj3Nmxg+uP5+mlk7+67PriRRkc+35kwd/oIikNK25J4CM9DT+500XsqC8mG88to6VOw6EXZKIxDmFe4LIzUrngWUXUz4+l688VMfm91rCLklE4pjCPYGMH5fF8lsXkZuZzrIHV/Fuc3vYJYlInFK4J5gp4/NYfusijnb2sPTBVTS3dYVdkojEIYV7Ajp3ciH3L13I7gNtfGV5HR3duh6KiJxM4Z6gFs+cyL2fm8+a3Yf4xqOv06tL9YlIPwr3BPaxCyZz18fn8sKGBv7LM2/rLFYROU597gnu5sum09DSyX0rtjOpMIdvXFUZdkkiEgcU7kngu9dU0XCkg5+8uIWMdOPmJRXkZel/WpFUpgRIAmbGD//6Ag63dfNPv9/MfbXb+dSFZdy0aCrnTi4MuzwRCYHFwzrtwoULva6uLuwyEp67s3rnIR5dtZvfvbWPrp4+5pcX8/lFU/n4vMmazYskGTNb4+4LB9yncE9OzW1dPLW2nkdW7WZbYysF2RncsCA2m597jmbzIslA4Z7C3J26XYd4ZKVm8yLJRuEuwMCz+esXnMPnF03TbF4kASnc5STHZvOPrtzNc8Fsfl55MZ9fVM7HLziHcdmazYskAoW7nFJzWxdPv17PIyt3s7WxlfzsDG5YcA43LZrKeecUhV2eiJzGiMLdzB4EPg40uvv5wdgE4HGgAtgJfNbdD5mZAT8FrgPagJvdfe1gBSrcw+furNl1iEdW7eZ3b+6js6ePeVOK+PwlUzWbF4lTIw33y4FW4KF+4f5PwEF3v8fM7gDGu/v3zOw64O+JhfslwE/d/ZLBClS4x5fDbd08/fpeHlm1my0Nsdn89fNjs/nzyzSbF4kXI16WMbMK4Ll+4b4ZqHb3fWY2GVjh7lVm9i/B9qPvP+50z69wj0/uztrdh3h45cmz+ZsWTeUT8zSbFwnbWIR7s7sXB9sGHHL3YjN7DrjH3V8O9v0B+J67fyC5zew24DaAqVOnXrRr167h/G1ylmg2LxJ/ThfuI556ubub2Rl/Kuvu9wP3Q2zmPtI6ZGwV5WVy82XTWbakgrW7D/HIyj08uWYvD6/czQVTivi8ZvMicWW4X/nbECzHENw2BuP1QHm/46YEY5IkzIyLpk3gx5+dx6o7r+YfP3kend193PHUWyy6+/9x59Nv8Xb94bDLFEl5w51mPQssA+4Jbp/pN/51M3uM2Aeqhwdbb5fEVZSXybIlFSxdPI21u5t5dNVunlq7l0dW7mbelCKWLq7gYxdMJiczPexSRVLOULplHgWqgRKgAbgL+DfgCWAqsItYK+TBYP39n4FribVC3jLQevv76QPV5HG4vZun1+7l16/tYnvTUSaMy+JzF5fzhUumMmV8XtjliSQVncQkZ52785ftB3jo1Z28uKEBgKvPLWXp4goumzWR2DxAREZiTD9QFRmImXHZrBIum1VCfXM7D7+2i8dW7+GFDQ3MjIxj6eIKPn1hGQU5mWGXKpKUNHOXs6aju5fn39rH8ld38caeZsZlpfPpC6ewdPE0KksLwi5PJOFoWUbizht7mnno1V383zffpaunj8UzJrJsyTSuPreUjHRdt11kKBTuErcOHu3i8dV7+M1ru6hvbmdyUQ5fuGQqNy6aSkl+dtjlicQ1hbvEvd4+54+bGnno1Z38eet+stLTuO5Dk1i6pIIF5cX6AFZkAPpAVeJeeprx0bmlfHRuKdsaW/nNa7t4cs1e/m3du5xfVsjSxRV8ct456pkXGSLN3CVutXb28PTr9fz61Z1saWilOC+Tzy0s54uXTqN8gnrmRbQsIwnN3Xltx0EeenUnL2xooM+dK6uiLF1SwUdmlZCWpiUbSU0Kd0ka+w6388jK3Ty6ajf7W7uYXjKOL146jb+5aApFueqZl9SicJek09nTy+/ffo/lf9nJ2t3N5Gam86kLy1i6eBpzJuli35IaFO6S1N7ae5iHXt3Js2+8S2dPH4umT2Dp4mlcc94kMtUzL0lM4S4p4dDRLp6o28OvX9vF3kPtRAuyWTxzIrNLC6iM5jO7tIDyCXmka41ekoTCXVJKb5+zYnMjj6/ew/p3j1Df3H58X3ZGGrOCoK8szWd2tICqSQWUFefqg1lJOOpzl5SSnmZcdW4pV51bCkBLRzfbGlvZ0tDClobY7avbD/D06yeuI5ObmU5laT6V0QJml54I/7LiXJ1AJQlJ4S5JryAnkwVTx7Ng6viTxg+3d7Ot8UTgb21o5c9bm/jt2r3HjxmXlc6s0gJm95/tlxYwuShHoS9xTeEuKasoN5OLpk3gomkTThpvbuvqF/ix8K/d3Mj/WXMi9AuyM44HfWXpidl+tCBboS9xQWvuIkN08GjXSYG/paGFrY2tHDzadfyYwpyMDwT+rGg+hTmZ5GSmKfhlVOkDVZExtL+1M7ae/14LWxpbj4f/4fbuDxybm5lOXlY6uVnBbeax7YzYbXA/tp1BXlY6OcH4scfFniPjA8+RnaE3j1SjD1RFxlBJfjYl+dksmVlyfMzdaWrpZEtDK+/sb6W1s5f2rh7au3tp6+qlvSu47Y5tN7Z00NbVS0dXL23BMV09fWdUR5pBXlYGOcEbwclvBulkZ6YTD9GfmZ5GYU4GhbmZFOZkUpibQdHx7RNjBTmZalsdAYW7yBgwM6KFOUQLc/hwZcngDxhAT29fLPy7T7wZtHX10tF9bLuH9uAN4oNvGD0nvXk0t3XT0dM7yn/l8HT19NHS0cORjm4GWzgoyI69CRSc9s3gxL6i3Nj+wtxM8rMyUrq9VeEuEqcy0tMoSE9L2uvM9vU5R7t6ONLRw5H2bg63d3Okvfv4/SMd3Rxpj70JHNtX39zOxn2xfS0dPad9frMTbw5F/d4Yjr0pTC7KOf7ZyKTC5Ot+UriLSCjS0oyCnEwKcjIpK84948f39jmtnf3eGPq9GZz0JtFv364DbcePP9p14r9kCnIyjp/FnCzdTwp3EUlI6WlGUTArLx/G4wfqfnphQwOPrd5z/Jii3Ewqo/lUlhZQ1a/1tSQ/K+5DX90yIiL9HOt+2trQyuZ+4d+/+2l8XuZJM/xjZzZPPMvX/VW3jIjIEA3W/RQ7vyEW+M+se/ektf+J47JOPrktWOoZPy7rrP8dCncRkUGcqvvJ3Wk4EpznEMz2tzS28NTaelo7T4R+SX728Vn+7GDGX1laMKYXmFG4i4gMk5kxqSiHSUU5XD47cnzc3Xn3cMdJa/pbG1p4om4Pbf0+yC0tzOZvPzKDr3xkxqjXNqJwN7OdQAvQC/S4+0IzmwA8DlQAO4HPuvuhkZUpIpI4zIyy4lzKinOpqYoeH+/rc+qb248v62xpaCFSMDbr9KMxc69x9/397t8B/MHd7zGzO4L73xuF3yMiktDS0ozyCXmUT8jjyjmlY/u7xuA5rweWB9vLgRvG4HeIiMhpjDTcHXjBzNaY2W3BWKm77wu23wMGfHsys9vMrM7M6pqamkZYhoiI9DfSZZkPu3u9mUWBF81sU/+d7u5mNmAjvbvfD9wPsT73EdYhIiL9jGjm7u71wW0j8DSwCGgws8kAwW3jSIsUEZEzM+xwN7NxZlZwbBv4D8DbwLPAsuCwZcAzIy1SRETOzEiWZUqBp4PvV8gAHnH335vZauAJM/sysAv47MjLFBGRMzHscHf3HcC8AcYPAFeNpCgRERmZsWiFFBGRkMXFt0KaWROxJZzhKAH2D3pU6tDrcTK9HifotThZMrwe09w9MtCOuAj3kTCzulN95WUq0utxMr0eJ+i1OFmyvx5alhERSUIKdxGRJJQM4X5/2AXEGb0eJ9PrcYJei5Ml9euR8GvuIiLyQckwcxcRkfdRuIuIJKGEDnczu9bMNpvZtuDCICnLzMrNrNbMNpjZejP7Ztg1hc3M0s3sdTN7LuxawmZmxWb2pJltMrONZrY47JrCYmb/Mfg38raZPWpmOWHXNBYSNtzNLB34X8BfAXOBm8xsbrhVhaoH+I67zwUuBW5P8dcD4JvAxrCLiBM/BX7v7nOIfW1ISr4uZlYGfANY6O7nA+nAjeFWNTYSNtyJfb3wNnff4e5dwGPErgKVktx9n7uvDbZbiP3jLQu3qvCY2RTgY8Avw64lbGZWBFwOPADg7l3u3hxqUeHKAHLNLAPIA94NuZ4xkcjhXgbs6Xd/LykcZv2ZWQWwAFgZcilh+h/Ad4G+kOuIB9OBJuBfg2WqXwZf051ygmtQ/AjYDewDDrv7C+FWNTYSOdxlAGaWD/wW+Ja7Hwm7njCY2ceBRndfE3YtcSIDuBC4z90XAEeJXbg+5ZjZeGL/hT8dOAcYZ2ZfDLeqsZHI4V4PlPe7PyUYS1lmlkks2B9296fCridElwGfNLOdxJbrrjSz34RbUqj2Anvd/dh/yT1JLOxT0dXAO+7e5O7dwFPAkpBrGhOJHO6rgUozm25mWcQ+FHk25JpCY7GrpjwAbHT3n4RdT5jc/fvuPsXdK4j9/+KP7p6Us7OhcPf3gD1mVhUMXQVsCLGkMO0GLjWzvODfzFUk6YfLI71AdmjcvcfMvg78O7FPvB909/UhlxWmy4AvAW+Z2bpg7E53fz68kiSO/D3wcDAR2gHcEnI9oXD3lWb2JLCWWIfZ6yTp1xDo6wdERJJQIi/LiIjIKSjcRUSSkMJdRCQJKdxFRJKQwl1EJAkp3EVEkpDCXUQkCf1/l0siOC6/dtUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "xs = []\n",
    "for i in range(10):\n",
    "    xs.append(len(tag_df[tag_df['count']>i][tag_df['count']<200]))\n",
    "pd.Series(xs).plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "id": "0b198bcc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zzbn1\\AppData\\Local\\Temp\\ipykernel_26468\\1747741503.py:1: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  tag_df[tag_df['count']>2][tag_df['count']<200]['count'].value_counts().sort_index().plot()\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAj2klEQVR4nO3de3hcd33n8fd3ZnSxJfkqyffY8TUkQBwjQoAkTciFxE8glHYhaUvDltbQhacNS0tpWUofyvayLNlewpKmJEvYpWkaSEhSDMRNaUJaICjBdhwc27KT4Ksk24kt2dZlpO/+cc6MZqQZaTQzuszR5/U8ejRzzpmZn47Hn/nN9/zO75i7IyIi0RWb6gaIiMjEUtCLiEScgl5EJOIU9CIiEaegFxGJuMRUNyCXxsZGX7Vq1VQ3Q0SkYjz77LPH3b0p17ppGfSrVq2itbV1qpshIlIxzOyVfOtUuhERiTgFvYhIxCnoRUQiTkEvIhJxCnoRkYgbM+jNbIWZfd/MfmZmL5jZ74bLF5jZNjPbF/6en+fxt4Xb7DOz28r9B4iIyOgK6dEngU+4+4XAZcBHzexC4FPAE+6+DngivJ/FzBYAnwXeAlwKfDbfB4KIiEyMMYPe3Y+6+3Ph7S5gN7AMuBm4L9zsPuA9OR7+TmCbu59091eBbcANZWj3aO3lwdaDnOsbmMiXERGpGOOq0ZvZKuAS4MfAInc/Gq46BizK8ZBlwMGM+4fCZbmee4uZtZpZa2dn53ialeWnB1/j97+xkyf3dhT9HCIiUVJw0JtZPfBN4HZ3P525zoOrl5R0BRN3v9vdW9y9pakp51m8BdlzrAuA3uRgKc0REYmMgoLezKoIQv7r7v5QuLjdzJaE65cAubrQh4EVGfeXh8smTCroBwZ15SwREShs1I0B9wC73f2OjFWPAqlRNLcBj+R4+PeA681sfngQ9vpw2YRJBX1SQS8iAhTWo3878AHgHWa2PfzZDPwFcJ2Z7QOuDe9jZi1m9hUAdz8J/Cnwk/Dnc+GyCbO3PQj6QQW9iAhQwOyV7v40YHlWX5Nj+1bgNzPu3wvcW2wDx+N4dy8nzvQB6tGLiKRE6szYvWHZBlSjFxFJiVTQ72lX0IuIDBepoN/b3kVtVfAnKehFRAKRCvo9x7rYsHgOoBq9iEhKZILe3dnb3s2FSxoAGHQFvYgIRCjoBwadz7/n9fzym5YDkBxQ0IuIwDS9OHgxEvEY77kkmEbHDAYGNQWCiAhEqEefKW7GgEo3IiJAVIM+ZjoYKyISimTQJ2LGgGr0IiJARIM+FlPpRkQkJZJBn4iZTpgSEQlFMujjsZhq9CIioYgGvaYpFhFJiWTQJ9SjFxFJi2TQx1WjFxFJi2zQq0cvIhIYcwoEM7sXuAnocPfXh8seADaEm8wDXnP3jTke+zLQBQwASXdvKUurxxCPmWr0IiKhQua6+SpwJ/C11AJ3f3/qtpl9ETg1yuOvdvfjxTawGImYkdRcNyIiQGHXjH3KzFblWmdmBrwPeEeZ21WSmKlGLyKSUmqN/gqg3d335VnvwONm9qyZbRnticxsi5m1mllrZ2dnSY1KxBX0IiIppQb9rcD9o6y/3N03ATcCHzWzK/Nt6O53u3uLu7c0NTWV1CgdjBURGVJ00JtZAngv8EC+bdz9cPi7A3gYuLTY1xuPuEo3IiJppfTorwVedPdDuVaaWZ2ZNaRuA9cDu0p4vYJpHL2IyJAxg97M7gd+CGwws0Nm9qFw1S0MK9uY2VIz2xreXQQ8bWY7gGeAb7v7d8vX9PxUoxcRGVLIqJtb8yz/YI5lR4DN4e0DwMUltq8oMVONXkQkJZJnxiZixqDmoxcRASIa9PFYjKSuMCUiAkQ26FGNXkQkFMmgT8RiupSgiEgokkGv4ZUiIkMiG/Sa1ExEJBDZoFfOi4gEIhn0mqZYRGRIJIM+phq9iEhaJIM+oaAXEUmLZNBrmmIRkSHRDHpNUywikhbNoNfslSIiaZEMetXoRUSGRDLo47GYavQiIqFoBr0ZAIMKexGRaAZ9Ih4EvXr1IiKFXUrwXjPrMLNdGcv+xMwOm9n28GdznsfeYGZ7zKzNzD5VzoaPJh4Lgl51ehGRwnr0XwVuyLH8f7n7xvBn6/CVZhYHvgTcCFwI3GpmF5bS2EKlSjeaqlhEpICgd/engJNFPPelQJu7H3D3PuAfgZuLeJ5xS/fodZUpEZGSavQfM7OdYWlnfo71y4CDGfcPhctyMrMtZtZqZq2dnZ0lNCuzRq+JzUREig36LwNrgI3AUeCLpTbE3e929xZ3b2lqairpuWKmGr2ISEpRQe/u7e4+4O6DwN8TlGmGOwysyLi/PFw24RIx1ehFRFKKCnozW5Jx9xeBXTk2+wmwzszON7Nq4Bbg0WJeb7xSNfqkavQiIiTG2sDM7geuAhrN7BDwWeAqM9sIOPAy8OFw26XAV9x9s7snzexjwPeAOHCvu78wEX/EcBpeKSIyZMygd/dbcyy+J8+2R4DNGfe3AiOGXk60uEo3IiJp0TwzNhb8WerRi4hENOjj4V+lGr2ISGSDPvizBlW6ERGJZtCnhldqUjMRkYgGfSw96kZnxoqIRDLo0ydMKedFRKIZ9OkTptSjFxGJdtBreKWIiIJeRCTyIhn0CQW9iEhaJIM+NU2xhleKiEQ06FMXHhlU0IuIRDTodcKUiEhaJINeV5gSERkSyaDX7JUiIkMiGfTxuHr0IiIp0Qx6jboREUkbM+jN7F4z6zCzXRnLvmBmL5rZTjN72Mzm5Xnsy2b2vJltN7PWMrZ7VLrClIjIkEJ69F8Fbhi2bBvwend/I7AX+MNRHn+1u29095bimjh+6ROmNKuZiMjYQe/uTwEnhy173N2T4d0fAcsnoG1Fi2l4pYhIWjlq9L8BfCfPOgceN7NnzWzLaE9iZlvMrNXMWjs7O0tqUKpHrytMiYiUGPRm9mkgCXw9zyaXu/sm4Ebgo2Z2Zb7ncve73b3F3VuamppKaVbGNMUKehGRooPezD4I3AT8qnvurrO7Hw5/dwAPA5cW+3rjkT4Yq4uDi4gUF/RmdgPwSeDd7n42zzZ1ZtaQug1cD+zKtW25pYZXatSNiEhhwyvvB34IbDCzQ2b2IeBOoAHYFg6dvCvcdqmZbQ0fugh42sx2AM8A33b3707IXzFMLGbETCdMiYgAJMbawN1vzbH4njzbHgE2h7cPABeX1LoSxGOmGr2ICBE9MxaCoNc0xSIiEQ76RCymHr2ICBEOetXoRUQCkQ36RDxGclBTIIiIRDbo4zFDU92IiEQ46BMxY0A9ehGR6AZ9zDS8UkQEIhz0ibiGV4qIQISDXidMiYgEohv0ZhpeKSJClIM+pqAXEYEIB30irqAXEYEIB31co25ERIAoB33MdClBEREiHPSJWIykrjAlIhLdoI/FNKmZiAhEOOgTsZguJSgiQoFBb2b3mlmHme3KWLbAzLaZ2b7w9/w8j70t3Gafmd1WroaPRSdMiYgECu3RfxW4YdiyTwFPuPs64InwfhYzWwB8FngLcCnw2XwfCOUW16RmIiJAgUHv7k8BJ4ctvhm4L7x9H/CeHA99J7DN3U+6+6vANkZ+YEwITVMsIhIopUa/yN2PhrePAYtybLMMOJhx/1C4bAQz22JmrWbW2tnZWUKzApqmWEQkUJaDse7uQEkFcXe/291b3L2lqamp5DbFVKMXEQFKC/p2M1sCEP7uyLHNYWBFxv3l4bIJl4hpmmIRESgt6B8FUqNobgMeybHN94DrzWx+eBD2+nDZhNOoGxGRQKHDK+8HfghsMLNDZvYh4C+A68xsH3BteB8zazGzrwC4+0ngT4GfhD+fC5dNOE1TLCISSBSykbvfmmfVNTm2bQV+M+P+vcC9RbWuBJq9UkQkENkzYzUfvYhIILpBr2mKRUSAKAd9LKZRNyIiRDjoE3H16EVEIMJBH9OoGxERIMJBn4iZpikWESHCQZ8adeMKexGZ4SId9KCrTImIRD/o1aMXkRkuskGfUI9eRASIcNCnevQaYikiM13kg35gQEEvIjNbZIM+oRq9iAgQ4aCPqUYvIgJEOOgTo9Toe5MD/Pl3dtPdm5zsZomITLoIB33wpyUHRl4g/PlDp/i7Jw/wzEsnJrtZIiKTLrJBX50I/rS+5MigP9c/kHediEjUFB30ZrbBzLZn/Jw2s9uHbXOVmZ3K2OaPS25xgWrCoO/NEea9/cGyPo3IEZEZoKBLCebi7nuAjQBmFgcOAw/n2PQH7n5Tsa9TrOpRgr4nGfTo+9WjF5EZoFylm2uA/e7+Spmer2SjlW560j16Bb2IRF+5gv4W4P48695qZjvM7DtmdlG+JzCzLWbWamatnZ2dJTeoJhEHghE2w/WENfp+Bb2IzAAlB72ZVQPvBh7Msfo5YKW7Xwz8LfCtfM/j7ne7e4u7tzQ1NZXarHSNPnePXgdjRWTmKEeP/kbgOXdvH77C3U+7e3d4eytQZWaNZXjNMaVLNzl67am6fb8OxorIDFCOoL+VPGUbM1tsZhbevjR8vUkZvJ4eddOfa9SNevQiMnMUPeoGwMzqgOuAD2cs+wiAu98F/DLw22aWBM4Bt/gkXfJptB59T7pHn73ueHcvNYkYDbVVE99AEZFJUlLQu/sZYOGwZXdl3L4TuLOU1yhWdXzsGv3woP+Nr/6Ei5bO4c/f+8aJb6CIyCQpKeins5qqsUfdDB9jf+S1HubPrp74xomITKLoToEwSo++N0/ppqunn66e/olvnIjIJIps0FfFg9krCy3d9CUH6U0O0tWjGS1FJFoiG/RmRk0ilnsKhP6RwytTUxZr6mIRiZrIBj0EI29yB/3I4ZWpko169CISNZEO+ppEbNThlZnrUgHf3ZvUValEJFIiHvTxUU+Y6s8R9ABn+tSrF5HoiHTQV+fp0afKOblKN8FtBb2IREe0gz4eo6/A2Sszw11DLEUkSiId9DVVYxyMzTHqBqBbPXoRiZBIB33Qox9leKVKNyIyA0Q76BMjg97d09Mi5Bp1A3BapRsRiZBIB32uE6b6B5zU6MmsGn1vkmBCZfXoRSRaIh30uXr0PRkHZ7NLN0maG2oAnR0rItES8aCPjxhemToQC8NLN/00NdQQj5lG3YhIpEQ66GsSsfTJUSmpE6jqaxJZvf3uniRzaquor0modCMikRLpoM91wlTqQGxDbSJrUrOuniQNtQkaahMaXikikVJy0JvZy2b2vJltN7PWHOvNzP7GzNrMbKeZbSr1NQtVHY+NmAIhNbQyCPrs0k19TdCjP62gF5EIKdcVpq529+N51t0IrAt/3gJ8Ofw94WqqYvTmqdHPqa0iOegMDjqxmNHVG/To59RWqUYvIpEyGaWbm4GveeBHwDwzWzIJr0tNeMJU5vXIM3v0EByQHRx0unuTzEmVbnKMutnf2c3OQ69NRrNFRMqqHEHvwONm9qyZbcmxfhlwMOP+oXBZFjPbYmatZtba2dlZhmYFNXrIHl2T6tE31FYBwVj6M31J3KG+NkF9be6DsR9/YDu//+DOsrRLRGQylSPoL3f3TQQlmo+a2ZXFPIm73+3uLe7e0tTUVIZmBdMUQ/YslakTqOpTPfrkYLoH31BbRUNtYkTp5qXjZ9h56BQdXT1laZeIyGQqOejd/XD4uwN4GLh02CaHgRUZ95eHyyZcukefHNmjn5Pu0Xu6Bx+MuqmiuzeZVe755x1HAHj1bD/JHNMei4hMZyUFvZnVmVlD6jZwPbBr2GaPAr8ejr65DDjl7kdLed1CpYI+cxqEnozhlRCUblI9+FSPvn/Asx7z2M4j6dsnz/ZNeLtFRMqp1B79IuBpM9sBPAN8292/a2YfMbOPhNtsBQ4AbcDfA/+lxNcsWE3OHn1we04Y9L3JwXSPvr4mQUNNsDw1sdmeY13sbe/mbWsWAnCiW0EvIpWlpOGV7n4AuDjH8rsybjvw0VJep1iFHoxNBf2csHQDwZmyzQ3w2I4jxAw+cNlK/mP/CQW9iFScSJ8ZmzoYm3nSVPpgbE1m6Sb7YCwEZ8q6O4/tPMLb1zayfnEDACfO9E5a+0VEyiHSQT/Uox+a76a3f4CaRCy9rn9gkO7eoExTX5tIfwB09SR5/vApXjlxlne9cSmNdcHMlsfVoxeRClOuM2Onpep4joOx/QPUVsWpyljX1ZMkZlBXHU+Xbrp6+vm3PR1UxY13XrSYObMSJGLGiW716EWkskQ66GuqcgX9ILVVMaoTwVVGUsMr62sSmFm6dHO8u5d/3nmUX1jfzNzZQfgvrK9WjV5EKk60SzfxHKNukkGPvjoe1O/7wx59qiefCvXPPPICx0738K6Lh2ZrWFhXoxq9iFScaPfo85wwVZOIURX26PvCcfSpnvyc2iru/JVLOPLaOWZVJ9j8hoygr69WjV5EKk7Egz4cdTNsCoTMGn1q1E0q6AFueuPSnM/XWF/DyyfOTGCLRUTKL9qlmzw9+tpEPKus09Xbny7djGZhnWr0IlJ5ZkjQDw2v7OkfpKYqc3il0x0ejB3LwvoazvYNcLZPFyYRkcoR6aCvyTXXzbDhlX3JgRGlm3wW1lcDmgZBRCpLpIM+V+mmNzkYHIyNZw+vLKR005gK+jMKehGpHJEO+kTMMMue66Y37NGnPgS6e5P0DQwW1qMPz47VSVMiUkkiHfRmRk0iNmya4uCEqapY8KefDHvnKt2ISFRFOughOGkq16ibWMyCKQ3CE6DG06M/rpOmRKSCRHocPUB1Ip7u0bt7+mBssC6W7tHX14xdo59VHaeuOl5Sj/6R7YepScS54fWLi36OSvOVHxzg2Kke1jbXs7a5njVN9cyvq57qZonMGJEP+qB0Ewyv3H20i0GH5fNnAVAVj42rdAOwZN4sfn7ybFFtefaVV/n4A9tJxGI88rG387olc4p6nkqy4+BrfP7bu4kZDA5dnZGFddWsyQj+1IfA0rm1mNnUNVgkgmZE0KdKN4/tPEIiZlx/UdCbLiboNyxq4IUjp8bdjrN9ST7xT9tZMncWvclBPv7Adh752NvTZ+9G1Re37WX+7Cqe/OTVnDrbT1tHd/pnf2c3W58/ymtnhy7GPrs6zpqmetY01aXDf21zPectqEsfQBeR8Sk66M1sBfA1gssJOnC3u//1sG2uAh4BXgoXPeTunyv2NYtRHQa9u/PYjiNcvq6RBWHZoDpuHDsd1ugLKN0ArF/UwNZdRznXN8Cs6sJD+s+27uaVk2e5/7cu40xvkg/d18pf/cs+/uCGC8b/R1WIZ146yVN7O/mjzRcwp7aKObVVrFgwm6svaE5v4+6cONOXFf5tHd0889JJvrV96Fq9iZhx3sLZrG3K/hawprm+oJPdRGayUv6HJIFPuPtz4QXCnzWzbe7+s2Hb/cDdbyrhdUqSGnWz/eBrHHr1HB+/dn16XXUili4nFNyjX1yPO7R1dPOG5XMLesyTezv5fz/6Ob91xflctjq49uwtb17B3z25n2suaKZl1YLx/VEVwN35n4/vobmhhg9ctirvdmZGY30NjfU16X2TcqY3yYHOM7R1dmV8EJzhX1/sIJlRB1oytzYr+FMfBo311SoDiVBC0Lv7UeBoeLvLzHYDy4DhQT+lUj36R3ccoToR47qLFqXXpc6OheDqUoVYvyi4pOCe9q6Cgv61s3188hs7WNdczyeu35Be/t9uupCn247ziQd3sPV3rqAuYr3Sf287wTMvneRzN180rm8+mepqErxh+dwR+7l/YJBXTpxNfwPY39FNW2c3D7Ye5Ezf0HQXc2oTWeWf1IfB8vmzicf0ASAzR1nSxcxWAZcAP86x+q1mtgM4Avyeu7+Q5zm2AFsAzjvvvHI0CwiC/kzvAN/eeZSrNzQxJ+MM2FTQ11bFskJ/NCsXBrXive1dBW3/mUde4ER3H/fc9ub0aB8Irll7x/s28v67f8h/37qbP/vFN4zjr5re3J0vPL6HZfNm8f43ryj781fFY+nwHv66x073jDgO8K8vdvJPrYfS21UnYqxurMvq/a9truf8xrqsfyORqCg56M2sHvgmcLu7nx62+jlgpbt3m9lm4FvAulzP4+53A3cDtLS0eK5tilGTiPPMSyfpH3DedXH29MOpg3uFTH+QEo8Z65rr2XNs7KB/dMcRHttxhN+7fj2vXzay93/p+QvYcsVq/u6pA1x34SKu3tCc41kqzxO7O9hx8DX+8pfeMKkHm82MJXNnsWTuLK5Y15S17tTZfto6u9jfcYa28DjA84dOsfX5o7inHg8r5s/O+AYQHhBuakhfkEakEpUU9GZWRRDyX3f3h4avzwx+d99qZv/bzBrd/Xgprzse1fEY/QPO7Oo477igecQ6KLw+n7JhUQM/PHBi1G3aT/fwmW/tYuOKeXzkF9bk3e7j163n3/Z08gff2Mn3br+y4seXDw46X9y2l1ULZ/PeTcunujlpc2dX8aaVC3jTyuzjIT39A7x0/MyIg8FPtx3POtGusb5mxEigNU31LNFwUKkApYy6MeAeYLe735Fnm8VAu7u7mV1KcCbu6AlZZqle+7WvW8Ts6uw/N3WVqYZx1sfXL27goZ8e5tS5fubOGtnTc3c++Y2d9CYHuON9F5MYpSxUWxXnjvdfzHu+9O985pFd3Pkrm8bVlunmO7uOsfvoaf7q/RsLLodNpdqqOK9bMmfEOQ0Dg86hV89mhX9bRzeP7TjC6Z6haarrquPpEtCajOMAKxfOroi/X2aGUnr0bwc+ADxvZtvDZX8EnAfg7ncBvwz8tpklgXPALe5etrJMIVJTFQ8v28BQjX48pRsIevQA+9q7skbMvHa2j73t3fz4wAme3NvJ526+iNVN9fmeJu2ipXO5/dr1fOF7e9i44gBvXD6PC5Y0ZB1PqAQDg84d2/awrrk+5/6uJPGYsXJhHSsX1nHN64YO4Ls7x7vD4aDhgeD9nd388MAJHvrp4fR2iZixqrEu+1tAUwOrm+oid+Bdpr9SRt08DYz6ndXd7wTuLPY1ymFBfTUL6qq5cn3jiHXFlm7WLw6C/sVj2UF/+wPb+bc9nQBcsa6RX3vLyoKf88NXruaJ3e18/tu7AbjpjUsqrnf/rZ8eZn/nGb78q5siO6rFzGhqqKGpoYa3rskeDtrdm0wHf+obwL6Obv5ldwcDGcNBl86tzXlW8MI6DQeViRH5rsXvvGMd//lt5+c8KFgV9vbHe8LN0rm11NckRoy8ef7QKa65oJkPXXE+b1o5n9g4wi4Rj/EPv3UZz73yKnd+v41dh8d/9u1U6h8Y5K+e2MtFS+fMqHl8MtXXJLh4xTwuXjEva3lfcpCfnzyTdS5AW0c3D/zkIGczhoPOm10VBH9T9nGA5fNnjeu9JDJc5IO+riaR96tydZGlGzNj/aLskTfHu3s5caaPt61t5G1rRn57KERtVZy3rW3kmZdP8tdP7Bv32bdT6cHWQxw8eY7/88HXq1c6THUixtrmBtY2N2QtHxx0jp7uCc4DCEtBbR3dPPFiOw+0HkxvV5OIsToV/k31rGmuSw8HjfoUGlIekQ/60RRbugHYsLiB7+46hrtjZuwNQz9Vvy/FhkUN4z77dir19A/wt/+6j03nzeOqDU1jP0AAiMWMZfNmsWzeLK5cn73fXj3Tly4BpX5vP/gq/7zzSHo4aMzgvAWzs88KDr8F5BokIDPXjA769KibIoJ+/aIG7n/mIJ3dvTQ31LInLOOsXzz2wdcxn3vx+M6+nWr3P/Nzjp7q4Yv/6WL15stkfl01LXULRkyPca4vHA6a+hAIvw38YN/xrCupNTXUjCgBrW2uZ9GcGv0bzUAzO+hL6dGHPfe9x7ppbqhlb3sX82dX0VRfU3K7Vi6YPa6zb6fS2b4kX/p+G29dvZC3rS2uZCWFm1Ud58Klc7hw6cjhoAdPns0aDdTW2c23th+mK2M4aENNgtVZJ4OlZgedPeowYKlsMzroi63RQ3av+/J1jew51sX6RQ1l6S0l4jHWNhV29u1Uu+8/XuF4dx93/dr6sTeWCRMPh3OuaqzjWrKHg3Z29Q6Ff/gB8B9tJ3jouaHhoFVxY9XCuhHfAFY31Y04/0Qqz4z+F6wuctQNBGdKLqyrZu+xLtydve3dvHfTsrK1bcPiBn40xtm3U+10Tz93PbmfqzY0RXIGzigwM5rn1NI8p3bEIIGunv70CKDUcYA9x7p4/GftWcNBl82bNWJeoDVNdSwsw7dXmRwzOuhLKd1AEMZ72rs4cqqH7t5kembLcli/qIGHRzn7djq49+mXOHWun09ct2HsjWXaaaitYuOKeWzMMRz0lRNDw0HbOoMPgvtfOsm5/qHhoPNnV2V9A0h9GCybp+Gg042CnuJKNxCE8YOtB3nxaDClz4bF5Qv6DeFB3eFn304Xr57p454fvMQNFy2uiAPGUrjqRIx1ixpYt2jkcNAjp85lnQuwv6Ob773QzskzQ8NBa6tirG6sz5oXKDUthIaDTo0ZHfRDs1cW36M/0zfAH3xzJ0BZe/QbFgcH2z72Dz8tun0Tqbs3SXdfko9fp9r8TBGLGcvnz2b5/NlcNexL3MmM4aCpUtBzP3+VR3cMXSUsHg4nrdElIfOaP7uaf/rIW8v+vNMvQSbR9Rcuoqunn+aG4mqN17yumfdesoye5ABrmxvKWmJZOreWD1+5moOvFnch8smw6bz5Zf0WI5VrQV01C+oW8OYcw0H3dw5dIOblE2dJDg7meRaZqPmtbJLnGCtIS0uLt7a2TnUzREQqhpk96+4tudbpO5SISMQp6EVEIk5BLyIScQp6EZGIU9CLiEScgl5EJOIU9CIiEaegFxGJuGl5wpSZdQKvjLFZI3B8EppTTmrz5Ki0Nldae0FtnizjafNKd895ibdpGfSFMLPWfGeBTVdq8+SotDZXWntBbZ4s5WqzSjciIhGnoBcRibhKDvq7p7oBRVCbJ0eltbnS2gtq82QpS5srtkYvIiKFqeQevYiIFEBBLyIScRUZ9GZ2g5ntMbM2M/vUVLcnFzNbYWbfN7OfmdkLZva74fI/MbPDZrY9/Nk81W1NMbOXzez5sF2t4bIFZrbNzPaFv+dPdTtTzGxDxn7cbmanzez26baPzexeM+sws10Zy3LuVwv8Tfje3mlmm6ZRm79gZi+G7XrYzOaFy1eZ2bmM/X3XNGpz3veCmf1huJ/3mNk7p0l7H8ho68tmtj1cXto+dveK+gHiwH5gNVAN7AAunOp25WjnEmBTeLsB2AtcCPwJ8HtT3b48bX4ZaBy27H8Anwpvfwr4y6lu5yjvi2PAyum2j4ErgU3ArrH2K7AZ+A5gwGXAj6dRm68HEuHtv8xo86rM7abZfs75Xgj/L+4AaoDzw0yJT3V7h63/IvDH5djHldijvxRoc/cD7t4H/CNw8xS3aQR3P+ruz4W3u4DdwLKpbVVRbgbuC2/fB7xn6poyqmuA/e4+1hnVk87dnwJODlucb7/eDHzNAz8C5pnZkklpaIZcbXb3x909Gd79EbB8sts1mjz7OZ+bgX909153fwloI8iWSTNae83MgPcB95fjtSox6JcBBzPuH2KaB6iZrQIuAX4cLvpY+PX33ulUCgEceNzMnjWzLeGyRe5+NLx9DFg0NU0b0y1k/6eYrvs4Jd9+rZT3928QfPNIOd/MfmpmT5rZFVPVqDxyvRem+36+Amh3930Zy4rex5UY9BXFzOqBbwK3u/tp4MvAGmAjcJTg69l0cbm7bwJuBD5qZldmrvTgO+S0G49rZtXAu4EHw0XTeR+PMF33az5m9mkgCXw9XHQUOM/dLwH+K/APZjZnqto3TEW9FzLcSnbHpaR9XIlBfxhYkXF/ebhs2jGzKoKQ/7q7PwTg7u3uPuDug8DfM8lfF0fj7ofD3x3AwwRta0+VDsLfHVPXwrxuBJ5z93aY3vs4Q779Oq3f32b2QeAm4FfDDyjC8seJ8PazBPXu9VPWyAyjvBem7X42swTwXuCB1LJS93ElBv1PgHVmdn7Yk7sFeHSK2zRCWGO7B9jt7ndkLM+st/4isGv4Y6eCmdWZWUPqNsGBt10E+/a2cLPbgEempoWjyur9TNd9PEy+/foo8Ovh6JvLgFMZJZ4pZWY3AJ8E3u3uZzOWN5lZPLy9GlgHHJiaVmYb5b3wKHCLmdWY2fkEbX5mstuXx7XAi+5+KLWg5H08mUeZy3i0ejPBKJb9wKenuj152ng5wdfxncD28Gcz8H+B58PljwJLprqtYXtXE4xC2AG8kNqvwELgCWAf8C/Agqlu67B21wEngLkZy6bVPib4EDoK9BPUgj+Ub78SjLb5Uvjefh5omUZtbiOoa6fez3eF2/5S+J7ZDjwHvGsatTnvewH4dLif9wA3Tof2hsu/Cnxk2LYl7WNNgSAiEnGVWLoREZFxUNCLiEScgl5EJOIU9CIiEaegFxGJOAW9iEjEKehFRCLu/wM/OTOpanEFqgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "tag_df[tag_df['count']>2][tag_df['count']<200]['count'].value_counts().sort_index().plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "853fefc5",
   "metadata": {},
   "source": [
    "A total of 366 tags remained in the embedding after filtering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "id": "0b78e6c8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zzbn1\\AppData\\Local\\Temp\\ipykernel_26468\\3256382964.py:1: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  tag_emb = tag_df[tag_df['count']>2][tag_df['count']<200]\n"
     ]
    }
   ],
   "source": [
    "tag_emb = tag_df[tag_df['count']>2][tag_df['count']<200]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "id": "e18c5065",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zzbn1\\AppData\\Local\\Temp\\ipykernel_26468\\1866734054.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  tag_emb['edge_l'] = tag_emb['origin'].apply(lambda x: list(x))\n"
     ]
    }
   ],
   "source": [
    "tag_emb['edge_l'] = tag_emb['origin'].apply(lambda x: list(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "id": "72ba8b15",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>origin</th>\n",
       "      <th>edge_l</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>garden</th>\n",
       "      <td>173</td>\n",
       "      <td>{3615, 3616, 4129, 3617, 3618, 3619, 3620, 362...</td>\n",
       "      <td>[3615, 3616, 4129, 3617, 3618, 3619, 3620, 362...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unescoworldheritage</th>\n",
       "      <td>144</td>\n",
       "      <td>{3615, 3616, 3617, 3618, 3619, 3620, 3621, 362...</td>\n",
       "      <td>[3615, 3616, 3617, 3618, 3619, 3620, 3621, 362...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>classicalgardens</th>\n",
       "      <td>133</td>\n",
       "      <td>{3615, 3616, 3617, 3618, 3619, 3620, 3621, 362...</td>\n",
       "      <td>[3615, 3616, 3617, 3618, 3619, 3620, 3621, 362...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>suzhoushi</th>\n",
       "      <td>60</td>\n",
       "      <td>{768, 781, 784, 785, 658, 787, 660, 4116, 4117...</td>\n",
       "      <td>[768, 781, 784, 785, 658, 787, 660, 4116, 4117...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>jiangsusheng</th>\n",
       "      <td>60</td>\n",
       "      <td>{768, 781, 784, 785, 658, 787, 660, 4116, 4117...</td>\n",
       "      <td>[768, 781, 784, 785, 658, 787, 660, 4116, 4117...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gate</th>\n",
       "      <td>3</td>\n",
       "      <td>{504, 3450, 2444}</td>\n",
       "      <td>[504, 3450, 2444]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lantern</th>\n",
       "      <td>3</td>\n",
       "      <td>{1018, 3004, 2446}</td>\n",
       "      <td>[1018, 3004, 2446]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>administrators</th>\n",
       "      <td>3</td>\n",
       "      <td>{2819, 395, 502}</td>\n",
       "      <td>[2819, 395, 502]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bw</th>\n",
       "      <td>3</td>\n",
       "      <td>{395, 621, 4013}</td>\n",
       "      <td>[395, 621, 4013]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fujifilmxf</th>\n",
       "      <td>3</td>\n",
       "      <td>{3369, 3370, 3373}</td>\n",
       "      <td>[3369, 3370, 3373]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>114 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    count                                             origin  \\\n",
       "garden                173  {3615, 3616, 4129, 3617, 3618, 3619, 3620, 362...   \n",
       "unescoworldheritage   144  {3615, 3616, 3617, 3618, 3619, 3620, 3621, 362...   \n",
       "classicalgardens      133  {3615, 3616, 3617, 3618, 3619, 3620, 3621, 362...   \n",
       "suzhoushi              60  {768, 781, 784, 785, 658, 787, 660, 4116, 4117...   \n",
       "jiangsusheng           60  {768, 781, 784, 785, 658, 787, 660, 4116, 4117...   \n",
       "...                   ...                                                ...   \n",
       "gate                    3                                  {504, 3450, 2444}   \n",
       "lantern                 3                                 {1018, 3004, 2446}   \n",
       "administrators          3                                   {2819, 395, 502}   \n",
       "bw                      3                                   {395, 621, 4013}   \n",
       "fujifilmxf              3                                 {3369, 3370, 3373}   \n",
       "\n",
       "                                                                edge_l  \n",
       "garden               [3615, 3616, 4129, 3617, 3618, 3619, 3620, 362...  \n",
       "unescoworldheritage  [3615, 3616, 3617, 3618, 3619, 3620, 3621, 362...  \n",
       "classicalgardens     [3615, 3616, 3617, 3618, 3619, 3620, 3621, 362...  \n",
       "suzhoushi            [768, 781, 784, 785, 658, 787, 660, 4116, 4117...  \n",
       "jiangsusheng         [768, 781, 784, 785, 658, 787, 660, 4116, 4117...  \n",
       "...                                                                ...  \n",
       "gate                                                 [504, 3450, 2444]  \n",
       "lantern                                             [1018, 3004, 2446]  \n",
       "administrators                                        [2819, 395, 502]  \n",
       "bw                                                    [395, 621, 4013]  \n",
       "fujifilmxf                                          [3369, 3370, 3373]  \n",
       "\n",
       "[114 rows x 3 columns]"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tag_emb"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "75563727",
   "metadata": {},
   "source": [
    "#### Constructing a Bipartite Graph for effective mapping of tag embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "995830ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "import networkx as nx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "61014639",
   "metadata": {},
   "outputs": [],
   "source": [
    "G=nx.from_dict_of_lists(tag_emb.to_dict()['edge_l'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "id": "ea4b7e18",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nx.is_bipartite(G)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "id": "49210f23",
   "metadata": {},
   "outputs": [],
   "source": [
    "emb_temp = nx.to_pandas_adjacency(G, dtype=int)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "370082cf",
   "metadata": {},
   "source": [
    "Filter out the tags with only one letter would result a tag embedding dimension of 107"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "id": "511cbc05",
   "metadata": {},
   "outputs": [],
   "source": [
    "tags = list(tag_emb.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "id": "768ac6d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "tags_n = [w for w in tags if len(w)>2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "id": "ec9d9609",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "107"
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pd.Series(tags_n).apply(lambda x : x.lower()).unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "id": "ca72a049",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['garden',\n",
       " 'unescoworldheritage',\n",
       " 'classicalgardens',\n",
       " 'suzhoushi',\n",
       " 'jiangsusheng',\n",
       " 'places',\n",
       " 'gardenofthehumbleadministrator',\n",
       " 'humble',\n",
       " 'administrator',\n",
       " 'zhu',\n",
       " 'lingering',\n",
       " 'at the',\n",
       " 'liongrove',\n",
       " 'culture',\n",
       " 'arts',\n",
       " 'centre',\n",
       " 'mercurytours',\n",
       " 'harryandrowenaphotos',\n",
       " 'street',\n",
       " 'temple',\n",
       " 'jeans',\n",
       " 'pagoda',\n",
       " 'shanghai',\n",
       " 'hangzhou',\n",
       " 'chinese',\n",
       " 'masterofnetsgarden',\n",
       " 'people',\n",
       " 'masterofthenets',\n",
       " 'north',\n",
       " 'central',\n",
       " 'square',\n",
       " 'jinji',\n",
       " 'lake',\n",
       " 'sony',\n",
       " 'plaza',\n",
       " 'pingjiang',\n",
       " 'center',\n",
       " 'couple',\n",
       " 'retreat',\n",
       " 'water',\n",
       " 'museum',\n",
       " 'city',\n",
       " 'house',\n",
       " 'pavilion',\n",
       " 'asia',\n",
       " 'canglang',\n",
       " 'keyuan',\n",
       " 'greatwave',\n",
       " 'photography',\n",
       " 'decoration',\n",
       " 'asian',\n",
       " 'december',\n",
       " 'peoples',\n",
       " 'prc',\n",
       " 'architecture',\n",
       " 'canal',\n",
       " 'republic',\n",
       " 'building',\n",
       " 'trees',\n",
       " 'candid',\n",
       " 'reflection',\n",
       " 'shantang',\n",
       " 'fuji',\n",
       " 'river',\n",
       " 'wall',\n",
       " 'red',\n",
       " 'landscape',\n",
       " 'pond',\n",
       " 'tree',\n",
       " 'east',\n",
       " 'green',\n",
       " 'art',\n",
       " 'plants',\n",
       " 'architectural',\n",
       " 'film',\n",
       " 'humbleadministratorgarden',\n",
       " 'tiger',\n",
       " 'vsco',\n",
       " 'province',\n",
       " 'venice',\n",
       " 'nature',\n",
       " 'hill',\n",
       " 'jiangnan',\n",
       " 'lotus',\n",
       " 'thelingeringgarden',\n",
       " 'fuji movie',\n",
       " 'liuyuan',\n",
       " 'unescoworldheritagesite',\n",
       " 'leafs',\n",
       " 'structures',\n",
       " 'family',\n",
       " 'canon',\n",
       " 'night',\n",
       " 'busy',\n",
       " 'waterway',\n",
       " 'pedro',\n",
       " 'man',\n",
       " 'China',\n",
       " 'electric',\n",
       " 'hotel',\n",
       " 'shops',\n",
       " 'monochrome',\n",
       " 'hanshan',\n",
       " 'gate',\n",
       " 'lantern',\n",
       " 'administrators',\n",
       " 'fujifilmxf']"
      ]
     },
     "execution_count": 184,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tags_n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "37301127",
   "metadata": {},
   "source": [
    "#### Tags embedding for data samples"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e439c56",
   "metadata": {},
   "source": [
    "As results, the 107-dimensional tags embedding could be mapped back to 1104 data points. Note that the duplicated data points in terms of text data are not considered."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "id": "06d8e271",
   "metadata": {},
   "outputs": [],
   "source": [
    "tags_emb = emb_temp.loc[tags_n].T.loc[~emb_temp.index.isin(tags)].sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "id": "9964b98c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>garden</th>\n",
       "      <th>unescoworldheritage</th>\n",
       "      <th>classicalgardens</th>\n",
       "      <th>suzhoushi</th>\n",
       "      <th>jiangsusheng</th>\n",
       "      <th>places</th>\n",
       "      <th>gardenofthehumbleadministrator</th>\n",
       "      <th>humble</th>\n",
       "      <th>administrator</th>\n",
       "      <th>zhu</th>\n",
       "      <th>...</th>\n",
       "      <th>China</th>\n",
       "      <th>electric</th>\n",
       "      <th>hotel</th>\n",
       "      <th>shops</th>\n",
       "      <th>monochrome</th>\n",
       "      <th>hanshan</th>\n",
       "      <th>gate</th>\n",
       "      <th>lantern</th>\n",
       "      <th>administrators</th>\n",
       "      <th>fujifilmxf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>108</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>363</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4082</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4111</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4116</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4117</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4129</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>555 rows × 107 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      garden  unescoworldheritage  classicalgardens  suzhoushi  jiangsusheng  \\\n",
       "108        1                    0                 0          0             0   \n",
       "132        1                    0                 0          0             0   \n",
       "135        1                    0                 0          0             0   \n",
       "361        0                    0                 0          0             0   \n",
       "363        0                    0                 0          0             0   \n",
       "...      ...                  ...               ...        ...           ...   \n",
       "4082       0                    0                 0          0             0   \n",
       "4111       0                    0                 0          0             0   \n",
       "4116       0                    0                 0          1             1   \n",
       "4117       0                    0                 0          1             1   \n",
       "4129       1                    0                 0          0             0   \n",
       "\n",
       "      places  gardenofthehumbleadministrator  humble  administrator  zhu  ...  \\\n",
       "108        0                               0       0              0    0  ...   \n",
       "132        0                               0       0              0    0  ...   \n",
       "135        0                               0       0              0    0  ...   \n",
       "361        0                               0       0              0    0  ...   \n",
       "363        0                               0       0              0    0  ...   \n",
       "...      ...                             ...     ...            ...  ...  ...   \n",
       "4082       1                               0       0              0    0  ...   \n",
       "4111       0                               0       0              0    0  ...   \n",
       "4116       0                               0       0              0    0  ...   \n",
       "4117       0                               0       0              0    0  ...   \n",
       "4129       0                               0       0              0    0  ...   \n",
       "\n",
       "      China  electric  hotel  shops  monochrome  hanshan  gate  lantern  \\\n",
       "108       1         0      0      0           0        0     0        0   \n",
       "132       0         0      0      0           0        0     0        0   \n",
       "135       0         0      0      0           0        0     0        0   \n",
       "361       0         0      0      0           0        0     0        0   \n",
       "363       0         0      0      0           0        0     0        0   \n",
       "...     ...       ...    ...    ...         ...      ...   ...      ...   \n",
       "4082      0         0      0      0           0        0     0        0   \n",
       "4111      0         0      0      0           0        1     0        0   \n",
       "4116      0         0      0      0           0        0     0        0   \n",
       "4117      0         0      0      0           0        0     0        0   \n",
       "4129      0         0      0      0           0        0     0        0   \n",
       "\n",
       "      administrators  fujifilmxf  \n",
       "108                0           0  \n",
       "132                0           0  \n",
       "135                0           0  \n",
       "361                0           0  \n",
       "363                0           0  \n",
       "...              ...         ...  \n",
       "4082               0           0  \n",
       "4111               0           0  \n",
       "4116               0           0  \n",
       "4117               0           0  \n",
       "4129               0           0  \n",
       "\n",
       "[555 rows x 107 columns]"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tags_emb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "id": "3d94d3d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "tags_VEC = tags_emb.apply(lambda x: np.array(x), axis=1).rename('VECTOR')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "id": "d26b6fef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "108     [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
       "132     [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
       "135     [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
       "361     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
       "363     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
       "                              ...                        \n",
       "4082    [0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
       "4111    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
       "4116    [0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
       "4117    [0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
       "4129    [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
       "Name: VECTOR, Length: 555, dtype: object"
      ]
     },
     "execution_count": 188,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tags_VEC"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "132b07d1",
   "metadata": {},
   "source": [
    "#### A subset with complet tag embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "id": "fdd55167",
   "metadata": {},
   "outputs": [],
   "source": [
    "sub = final_df.merge(tags_VEC, how='right', left_index=True, right_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "id": "f0453ede",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>candownload</th>\n",
       "      <th>url_c</th>\n",
       "      <th>url_q</th>\n",
       "      <th>url_n</th>\n",
       "      <th>url_largest</th>\n",
       "      <th>owner</th>\n",
       "      <th>owner_loc</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>...</th>\n",
       "      <th>week</th>\n",
       "      <th>unique_week</th>\n",
       "      <th>Chinese</th>\n",
       "      <th>English</th>\n",
       "      <th>Other_Lang</th>\n",
       "      <th>num_sent</th>\n",
       "      <th>revised_text</th>\n",
       "      <th>text_data</th>\n",
       "      <th>text_len</th>\n",
       "      <th>VECTOR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>108</th>\n",
       "      <td>51273169730</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5127316973...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5127316973...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5127316973...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5127316973...</td>\n",
       "      <td>63829447@N04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Couple Garden</td>\n",
       "      <td>...</td>\n",
       "      <td>41</td>\n",
       "      <td>201041</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>It has tags of china, china, chinese, garden, ...</td>\n",
       "      <td>Couple Garden. It has title of . It has tags o...</td>\n",
       "      <td>10.0</td>\n",
       "      <td>[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>51630786278</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5163078627...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5163078627...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5163078627...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5163078627...</td>\n",
       "      <td>193550492@N08</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Tetrahedral shadow and rockery</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>26</td>\n",
       "      <td>201526</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>It has title of Tetrahedral shadow and rockery...</td>\n",
       "      <td>. It has title of Tetrahedral shadow and rocke...</td>\n",
       "      <td>13.0</td>\n",
       "      <td>[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>51399368631</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5139936863...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5139936863...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5139936863...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5139936863...</td>\n",
       "      <td>193550492@N08</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Scene in a Circle in a Circle</td>\n",
       "      <td>Chinese Gardening</td>\n",
       "      <td>...</td>\n",
       "      <td>26</td>\n",
       "      <td>201526</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>It has title of Scene in a Circle in a Circle....</td>\n",
       "      <td>Chinese Gardening. It has title of Scene in a ...</td>\n",
       "      <td>16.0</td>\n",
       "      <td>[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361</th>\n",
       "      <td>51089471766</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5108947176...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5108947176...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5108947176...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5108947176...</td>\n",
       "      <td>44934278@N04</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>Canglang Pavilion 沧浪亭</td>\n",
       "      <td>The Canglang Pavilion (沧浪亭), variously transla...</td>\n",
       "      <td>...</td>\n",
       "      <td>12</td>\n",
       "      <td>202112</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>The Canglang Pavilion (沧浪亭), variously transla...</td>\n",
       "      <td>The Canglang Pavilion (沧浪亭), variously transla...</td>\n",
       "      <td>60.0</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>363</th>\n",
       "      <td>51089471666</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/65535/5108947166...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5108947166...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5108947166...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5108947166...</td>\n",
       "      <td>44934278@N04</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>Master of the Nets Garden 网师园</td>\n",
       "      <td>The Master of the Nets Garden (Chinese: 网师园) i...</td>\n",
       "      <td>...</td>\n",
       "      <td>12</td>\n",
       "      <td>202112</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>The Master of the Nets Garden (Chinese: 网师园) i...</td>\n",
       "      <td>The Master of the Nets Garden (Chinese: 网师园) i...</td>\n",
       "      <td>93.0</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4082</th>\n",
       "      <td>41857278401</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/958/41857278401_...</td>\n",
       "      <td>https://live.staticflickr.com/958/41857278401_...</td>\n",
       "      <td>https://live.staticflickr.com/958/41857278401_...</td>\n",
       "      <td>https://live.staticflickr.com/958/41857278401_...</td>\n",
       "      <td>7287512@N02</td>\n",
       "      <td>Kokomo, IN, USA</td>\n",
       "      <td>DSC_20180421_10836</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>16</td>\n",
       "      <td>201816</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>It has tags of china, places</td>\n",
       "      <td>. It has title of DSC_20180421_10836. It has t...</td>\n",
       "      <td>6.0</td>\n",
       "      <td>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4111</th>\n",
       "      <td>27910400829</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/4751/27910400829...</td>\n",
       "      <td>https://live.staticflickr.com/4751/27910400829...</td>\n",
       "      <td>https://live.staticflickr.com/4751/27910400829...</td>\n",
       "      <td>https://live.staticflickr.com/4751/27910400829...</td>\n",
       "      <td>127423136@N05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Corridor, Hanshan Temple - Suzhou, China</td>\n",
       "      <td>OLYMPUS DIGITAL CAMERA</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>201801</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>It has title of Corridor, Hanshan Temple - Suz...</td>\n",
       "      <td>OLYMPUS DIGITAL CAMERA. It has title of Corrid...</td>\n",
       "      <td>19.0</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4116</th>\n",
       "      <td>38919131931</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/4544/38919131931...</td>\n",
       "      <td>https://live.staticflickr.com/4544/38919131931...</td>\n",
       "      <td>https://live.staticflickr.com/4544/38919131931...</td>\n",
       "      <td>https://live.staticflickr.com/4544/38919131931...</td>\n",
       "      <td>95645163@N00</td>\n",
       "      <td>Milan</td>\n",
       "      <td>IMG_6925</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>34</td>\n",
       "      <td>201734</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>It has tags of suzhoushi, jiangsusheng, china, cn</td>\n",
       "      <td>. It has title of IMG_6925. It has tags of suz...</td>\n",
       "      <td>8.0</td>\n",
       "      <td>[0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4117</th>\n",
       "      <td>27142771579</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/4647/27142771579...</td>\n",
       "      <td>https://live.staticflickr.com/4647/27142771579...</td>\n",
       "      <td>https://live.staticflickr.com/4647/27142771579...</td>\n",
       "      <td>https://live.staticflickr.com/4647/27142771579...</td>\n",
       "      <td>95645163@N00</td>\n",
       "      <td>Milan</td>\n",
       "      <td>IMG_6928</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>34</td>\n",
       "      <td>201734</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>It has tags of suzhoushi, jiangsusheng, china, cn</td>\n",
       "      <td>. It has title of IMG_6928. It has tags of suz...</td>\n",
       "      <td>8.0</td>\n",
       "      <td>[0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4129</th>\n",
       "      <td>38689625892</td>\n",
       "      <td>1.0</td>\n",
       "      <td>https://live.staticflickr.com/4568/38689625892...</td>\n",
       "      <td>https://live.staticflickr.com/4568/38689625892...</td>\n",
       "      <td>https://live.staticflickr.com/4568/38689625892...</td>\n",
       "      <td>https://live.staticflickr.com/4568/38689625892...</td>\n",
       "      <td>147141688@N03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pavilion of Surging Waves</td>\n",
       "      <td>Canglang Pavilion 滄浪亭\\n\\n&lt;a href=\"https://en.w...</td>\n",
       "      <td>...</td>\n",
       "      <td>48</td>\n",
       "      <td>201748</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>It has title of Pavilion of Surging Waves. It ...</td>\n",
       "      <td>Canglang Pavilion 滄浪亭\\n\\n&lt;a href=\"https://en.w...</td>\n",
       "      <td>20.0</td>\n",
       "      <td>[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>555 rows × 35 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            index  candownload  \\\n",
       "108   51273169730          1.0   \n",
       "132   51630786278          1.0   \n",
       "135   51399368631          1.0   \n",
       "361   51089471766          1.0   \n",
       "363   51089471666          1.0   \n",
       "...           ...          ...   \n",
       "4082  41857278401          1.0   \n",
       "4111  27910400829          1.0   \n",
       "4116  38919131931          1.0   \n",
       "4117  27142771579          1.0   \n",
       "4129  38689625892          1.0   \n",
       "\n",
       "                                                  url_c  \\\n",
       "108   https://live.staticflickr.com/65535/5127316973...   \n",
       "132   https://live.staticflickr.com/65535/5163078627...   \n",
       "135   https://live.staticflickr.com/65535/5139936863...   \n",
       "361   https://live.staticflickr.com/65535/5108947176...   \n",
       "363   https://live.staticflickr.com/65535/5108947166...   \n",
       "...                                                 ...   \n",
       "4082  https://live.staticflickr.com/958/41857278401_...   \n",
       "4111  https://live.staticflickr.com/4751/27910400829...   \n",
       "4116  https://live.staticflickr.com/4544/38919131931...   \n",
       "4117  https://live.staticflickr.com/4647/27142771579...   \n",
       "4129  https://live.staticflickr.com/4568/38689625892...   \n",
       "\n",
       "                                                  url_q  \\\n",
       "108   https://live.staticflickr.com/65535/5127316973...   \n",
       "132   https://live.staticflickr.com/65535/5163078627...   \n",
       "135   https://live.staticflickr.com/65535/5139936863...   \n",
       "361   https://live.staticflickr.com/65535/5108947176...   \n",
       "363   https://live.staticflickr.com/65535/5108947166...   \n",
       "...                                                 ...   \n",
       "4082  https://live.staticflickr.com/958/41857278401_...   \n",
       "4111  https://live.staticflickr.com/4751/27910400829...   \n",
       "4116  https://live.staticflickr.com/4544/38919131931...   \n",
       "4117  https://live.staticflickr.com/4647/27142771579...   \n",
       "4129  https://live.staticflickr.com/4568/38689625892...   \n",
       "\n",
       "                                                  url_n  \\\n",
       "108   https://live.staticflickr.com/65535/5127316973...   \n",
       "132   https://live.staticflickr.com/65535/5163078627...   \n",
       "135   https://live.staticflickr.com/65535/5139936863...   \n",
       "361   https://live.staticflickr.com/65535/5108947176...   \n",
       "363   https://live.staticflickr.com/65535/5108947166...   \n",
       "...                                                 ...   \n",
       "4082  https://live.staticflickr.com/958/41857278401_...   \n",
       "4111  https://live.staticflickr.com/4751/27910400829...   \n",
       "4116  https://live.staticflickr.com/4544/38919131931...   \n",
       "4117  https://live.staticflickr.com/4647/27142771579...   \n",
       "4129  https://live.staticflickr.com/4568/38689625892...   \n",
       "\n",
       "                                            url_largest          owner  \\\n",
       "108   https://live.staticflickr.com/65535/5127316973...   63829447@N04   \n",
       "132   https://live.staticflickr.com/65535/5163078627...  193550492@N08   \n",
       "135   https://live.staticflickr.com/65535/5139936863...  193550492@N08   \n",
       "361   https://live.staticflickr.com/65535/5108947176...   44934278@N04   \n",
       "363   https://live.staticflickr.com/65535/5108947166...   44934278@N04   \n",
       "...                                                 ...            ...   \n",
       "4082  https://live.staticflickr.com/958/41857278401_...    7287512@N02   \n",
       "4111  https://live.staticflickr.com/4751/27910400829...  127423136@N05   \n",
       "4116  https://live.staticflickr.com/4544/38919131931...   95645163@N00   \n",
       "4117  https://live.staticflickr.com/4647/27142771579...   95645163@N00   \n",
       "4129  https://live.staticflickr.com/4568/38689625892...  147141688@N03   \n",
       "\n",
       "            owner_loc                                     title  \\\n",
       "108               NaN                                       NaN   \n",
       "132               NaN            Tetrahedral shadow and rockery   \n",
       "135               NaN             Scene in a Circle in a Circle   \n",
       "361    Beijing, China                     Canglang Pavilion 沧浪亭   \n",
       "363    Beijing, China             Master of the Nets Garden 网师园   \n",
       "...               ...                                       ...   \n",
       "4082  Kokomo, IN, USA                        DSC_20180421_10836   \n",
       "4111              NaN  Corridor, Hanshan Temple - Suzhou, China   \n",
       "4116            Milan                                  IMG_6925   \n",
       "4117            Milan                                  IMG_6928   \n",
       "4129              NaN                 Pavilion of Surging Waves   \n",
       "\n",
       "                                            description  ...  week  \\\n",
       "108                                       Couple Garden  ...    41   \n",
       "132                                                 NaN  ...    26   \n",
       "135                                   Chinese Gardening  ...    26   \n",
       "361   The Canglang Pavilion (沧浪亭), variously transla...  ...    12   \n",
       "363   The Master of the Nets Garden (Chinese: 网师园) i...  ...    12   \n",
       "...                                                 ...  ...   ...   \n",
       "4082                                                NaN  ...    16   \n",
       "4111                             OLYMPUS DIGITAL CAMERA  ...     1   \n",
       "4116                                                NaN  ...    34   \n",
       "4117                                                NaN  ...    34   \n",
       "4129  Canglang Pavilion 滄浪亭\\n\\n<a href=\"https://en.w...  ...    48   \n",
       "\n",
       "     unique_week  Chinese  English Other_Lang  num_sent  \\\n",
       "108       201041      0.0      0.0        1.0       1.0   \n",
       "132       201526      0.0      1.0        1.0       2.0   \n",
       "135       201526      0.0      1.0        1.0       2.0   \n",
       "361       202112      0.0      1.0        1.0       4.0   \n",
       "363       202112      0.0      1.0        1.0       6.0   \n",
       "...          ...      ...      ...        ...       ...   \n",
       "4082      201816      0.0      0.0        1.0       1.0   \n",
       "4111      201801      0.0      1.0        0.0       2.0   \n",
       "4116      201734      0.0      0.0        1.0       1.0   \n",
       "4117      201734      0.0      0.0        1.0       1.0   \n",
       "4129      201748      0.0      1.0        1.0       3.0   \n",
       "\n",
       "                                           revised_text  \\\n",
       "108   It has tags of china, china, chinese, garden, ...   \n",
       "132   It has title of Tetrahedral shadow and rockery...   \n",
       "135   It has title of Scene in a Circle in a Circle....   \n",
       "361   The Canglang Pavilion (沧浪亭), variously transla...   \n",
       "363   The Master of the Nets Garden (Chinese: 网师园) i...   \n",
       "...                                                 ...   \n",
       "4082                       It has tags of china, places   \n",
       "4111  It has title of Corridor, Hanshan Temple - Suz...   \n",
       "4116  It has tags of suzhoushi, jiangsusheng, china, cn   \n",
       "4117  It has tags of suzhoushi, jiangsusheng, china, cn   \n",
       "4129  It has title of Pavilion of Surging Waves. It ...   \n",
       "\n",
       "                                              text_data text_len  \\\n",
       "108   Couple Garden. It has title of . It has tags o...     10.0   \n",
       "132   . It has title of Tetrahedral shadow and rocke...     13.0   \n",
       "135   Chinese Gardening. It has title of Scene in a ...     16.0   \n",
       "361   The Canglang Pavilion (沧浪亭), variously transla...     60.0   \n",
       "363   The Master of the Nets Garden (Chinese: 网师园) i...     93.0   \n",
       "...                                                 ...      ...   \n",
       "4082  . It has title of DSC_20180421_10836. It has t...      6.0   \n",
       "4111  OLYMPUS DIGITAL CAMERA. It has title of Corrid...     19.0   \n",
       "4116  . It has title of IMG_6925. It has tags of suz...      8.0   \n",
       "4117  . It has title of IMG_6928. It has tags of suz...      8.0   \n",
       "4129  Canglang Pavilion 滄浪亭\\n\\n<a href=\"https://en.w...     20.0   \n",
       "\n",
       "                                                 VECTOR  \n",
       "108   [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "132   [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "135   [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "361   [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "363   [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "...                                                 ...  \n",
       "4082  [0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "4111  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "4116  [0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "4117  [0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "4129  [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "\n",
       "[555 rows x 35 columns]"
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "92d73031",
   "metadata": {},
   "source": [
    "### Save the Datasets generated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "id": "23dd909e",
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences.to_csv('Suzhou/data_storage/sentences.csv',sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "id": "21e22c04",
   "metadata": {},
   "outputs": [],
   "source": [
    "candidates_new.to_csv('Suzhou/data_storage/metadata_unique_text.csv',sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "id": "acda958c",
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df.to_csv('Suzhou/data_storage/metadata.csv',sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "id": "714544bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "tags_emb.to_csv('Suzhou/data_storage/tags_embedding.csv',sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "id": "5cc22c2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "sub.to_csv('Suzhou/data_storage/meta_with_tags.csv',sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c55b913d",
   "metadata": {},
   "source": [
    "### Comparing the Tag Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "id": "55098b7d",
   "metadata": {},
   "outputs": [],
   "source": [
    "tags_emb_V = pd.read_csv('Venezia/data_storage/tags_embedding.csv',sep='\\t', index_col='Unnamed: 0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "id": "ad47fea0",
   "metadata": {},
   "outputs": [],
   "source": [
    "tags_V = set(tags_emb_V.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "id": "746f331c",
   "metadata": {},
   "outputs": [],
   "source": [
    "tags_S = set(tags_n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "id": "9e61f2a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'architecture',\n",
       " 'art',\n",
       " 'building',\n",
       " 'canal',\n",
       " 'canon',\n",
       " 'city',\n",
       " 'family',\n",
       " 'film',\n",
       " 'fuji movie',\n",
       " 'garden',\n",
       " 'green',\n",
       " 'house',\n",
       " 'landscape',\n",
       " 'man',\n",
       " 'monochrome',\n",
       " 'museum',\n",
       " 'nature',\n",
       " 'night',\n",
       " 'people',\n",
       " 'photography',\n",
       " 'places',\n",
       " 'red',\n",
       " 'reflection',\n",
       " 'river',\n",
       " 'sony',\n",
       " 'square',\n",
       " 'street',\n",
       " 'unescoworldheritagesite',\n",
       " 'wall',\n",
       " 'water'}"
      ]
     },
     "execution_count": 201,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tags_V.intersection(tags_S)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8df6ef6d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (env)",
   "language": "python",
   "name": "env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}