0917_필사과제.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "0917 필사과제.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyOOoU/8L62CIKLo5zut8xZU",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jaesunkkk/esaa_assignment/blob/main/0917_%ED%95%84%EC%82%AC%EA%B3%BC%EC%A0%9C.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y_O0J71rRPhn",
        "outputId": "353854ed-93ff-4d31-866e-2931e7a1b1c6"
      },
      "source": [
        "from sklearn.datasets import fetch_20newsgroups\n",
        "\n",
        "news_data = fetch_20newsgroups(subset='all',random_state=156)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Downloading 20news dataset. This may take a few minutes.\n",
            "Downloading dataset from https://ndownloader.figshare.com/files/5975967 (14 MB)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tHcU8i-9RSn4",
        "outputId": "bcb8aae3-9891-463b-930d-11fcb36fae0f"
      },
      "source": [
        "print(news_data.keys())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "dict_keys(['data', 'filenames', 'target_names', 'target', 'DESCR'])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AeFvJzBLRT6G",
        "outputId": "b40ede93-f3d3-43a2-8b0a-048c2e03a4c4"
      },
      "source": [
        "import pandas as pd\n",
        "\n",
        "print('target 클래스의 값과 분포도 \\n', pd.Series(news_data.target).value_counts().sort_index())\n",
        "print('target 클래스의 이름들 \\n', news_data.target_names)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "target 클래스의 값과 분포도 \n",
            " 0     799\n",
            "1     973\n",
            "2     985\n",
            "3     982\n",
            "4     963\n",
            "5     988\n",
            "6     975\n",
            "7     990\n",
            "8     996\n",
            "9     994\n",
            "10    999\n",
            "11    991\n",
            "12    984\n",
            "13    990\n",
            "14    987\n",
            "15    997\n",
            "16    910\n",
            "17    940\n",
            "18    775\n",
            "19    628\n",
            "dtype: int64\n",
            "target 클래스의 이름들 \n",
            " ['alt.atheism', 'comp.graphics', 'comp.os.ms-windows.misc', 'comp.sys.ibm.pc.hardware', 'comp.sys.mac.hardware', 'comp.windows.x', 'misc.forsale', 'rec.autos', 'rec.motorcycles', 'rec.sport.baseball', 'rec.sport.hockey', 'sci.crypt', 'sci.electronics', 'sci.med', 'sci.space', 'soc.religion.christian', 'talk.politics.guns', 'talk.politics.mideast', 'talk.politics.misc', 'talk.religion.misc']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6WeJuoXSRWh9",
        "outputId": "b6d9f878-0462-44f8-c9b8-08cfe6b4f99d"
      },
      "source": [
        "print(news_data.data[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "From: egreen@east.sun.com (Ed Green - Pixel Cruncher)\n",
            "Subject: Re: Observation re: helmets\n",
            "Organization: Sun Microsystems, RTP, NC\n",
            "Lines: 21\n",
            "Distribution: world\n",
            "Reply-To: egreen@east.sun.com\n",
            "NNTP-Posting-Host: laser.east.sun.com\n",
            "\n",
            "In article 211353@mavenry.altcit.eskimo.com, maven@mavenry.altcit.eskimo.com (Norman Hamer) writes:\n",
            "> \n",
            "> The question for the day is re: passenger helmets, if you don't know for \n",
            ">certain who's gonna ride with you (like say you meet them at a .... church \n",
            ">meeting, yeah, that's the ticket)... What are some guidelines? Should I just \n",
            ">pick up another shoei in my size to have a backup helmet (XL), or should I \n",
            ">maybe get an inexpensive one of a smaller size to accomodate my likely \n",
            ">passenger? \n",
            "\n",
            "If your primary concern is protecting the passenger in the event of a\n",
            "crash, have him or her fitted for a helmet that is their size.  If your\n",
            "primary concern is complying with stupid helmet laws, carry a real big\n",
            "spare (you can put a big or small head in a big helmet, but not in a\n",
            "small one).\n",
            "\n",
            "---\n",
            "Ed Green, former Ninjaite |I was drinking last night with a biker,\n",
            "  Ed.Green@East.Sun.COM   |and I showed him a picture of you.  I said,\n",
            "DoD #0111  (919)460-8302  |\"Go on, get to know her, you'll like her!\"\n",
            " (The Grateful Dead) -->  |It seemed like the least I could do...\n",
            "\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xUF-9quBRabC",
        "outputId": "92f7b0cf-19c7-42ec-dfd4-696eaecd2289"
      },
      "source": [
        "from sklearn.datasets import fetch_20newsgroups\n",
        "\n",
        "# subset='train'으로 학습용(Train) 데이터만 추출, remove=('headers', 'footers', 'quotes')로 내용만 추출\n",
        "train_news= fetch_20newsgroups(subset='train', remove=('headers', 'footers', 'quotes'), random_state=156)\n",
        "X_train = train_news.data\n",
        "y_train = train_news.target\n",
        "print(type(X_train))\n",
        "\n",
        "# subset='test'으로 테스트(Test) 데이터만 추출, remove=('headers', 'footers', 'quotes')로 내용만 추출\n",
        "test_news= fetch_20newsgroups(subset='test',remove=('headers', 'footers','quotes'),random_state=156)\n",
        "X_test = test_news.data\n",
        "y_test = test_news.target\n",
        "print('학습 데이터 크기 {0} , 테스트 데이터 크기 {1}'.format(len(train_news.data) , len(test_news.data)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'list'>\n",
            "학습 데이터 크기 11314 , 테스트 데이터 크기 7532\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YMicnwpURcc-",
        "outputId": "42d89c7d-d742-42db-f241-4103b7ec0735"
      },
      "source": [
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "\n",
        "# Count Vectorization으로 feature extraction 변환 수행. \n",
        "cnt_vect = CountVectorizer()\n",
        "# 개정판 소스 코드 변경(2019.12.24)\n",
        "cnt_vect.fit(X_train)\n",
        "X_train_cnt_vect = cnt_vect.transform(X_train)\n",
        "\n",
        "# 학습 데이터로 fit( )된 CountVectorizer를 이용하여 테스트 데이터를 feature extraction 변환 수행. \n",
        "X_test_cnt_vect = cnt_vect.transform(X_test)\n",
        "\n",
        "print('학습 데이터 Text의 CountVectorizer Shape:',X_train_cnt_vect.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "학습 데이터 Text의 CountVectorizer Shape: (11314, 101631)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qwdKnOZIReuh",
        "outputId": "f4428076-f69e-4915-c4a0-e126c0c2423e"
      },
      "source": [
        "\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.metrics import accuracy_score\n",
        "\n",
        "# LogisticRegression을 이용하여 학습/예측/평가 수행. \n",
        "lr_clf = LogisticRegression()\n",
        "lr_clf.fit(X_train_cnt_vect , y_train)\n",
        "pred = lr_clf.predict(X_test_cnt_vect)\n",
        "print('CountVectorized Logistic Regression 의 예측 정확도는 {0:.3f}'.format(accuracy_score(y_test,pred)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CountVectorized Logistic Regression 의 예측 정확도는 0.608\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JWmgvWmxRg4V",
        "outputId": "a57ebb40-f29e-4c05-d79b-df8b8d3b6248"
      },
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "\n",
        "# TF-IDF Vectorization 적용하여 학습 데이터셋과 테스트 데이터 셋 변환. \n",
        "tfidf_vect = TfidfVectorizer()\n",
        "tfidf_vect.fit(X_train)\n",
        "X_train_tfidf_vect = tfidf_vect.transform(X_train)\n",
        "X_test_tfidf_vect = tfidf_vect.transform(X_test)\n",
        "\n",
        "# LogisticRegression을 이용하여 학습/예측/평가 수행. \n",
        "lr_clf = LogisticRegression()\n",
        "lr_clf.fit(X_train_tfidf_vect , y_train)\n",
        "pred = lr_clf.predict(X_test_tfidf_vect)\n",
        "print('TF-IDF Logistic Regression 의 예측 정확도는 {0:.3f}'.format(accuracy_score(y_test ,pred)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "TF-IDF Logistic Regression 의 예측 정확도는 0.674\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "75q60jlpRi4r",
        "outputId": "b8e62289-a2ef-4b57-b28e-56120c74524d"
      },
      "source": [
        "# stop words 필터링을 추가하고 ngram을 기본(1,1)에서 (1,2)로 변경하여 Feature Vectorization 적용.\n",
        "tfidf_vect = TfidfVectorizer(stop_words='english', ngram_range=(1,2), max_df=300 )\n",
        "tfidf_vect.fit(X_train)\n",
        "X_train_tfidf_vect = tfidf_vect.transform(X_train)\n",
        "X_test_tfidf_vect = tfidf_vect.transform(X_test)\n",
        "\n",
        "lr_clf = LogisticRegression()\n",
        "lr_clf.fit(X_train_tfidf_vect , y_train)\n",
        "pred = lr_clf.predict(X_test_tfidf_vect)\n",
        "print('TF-IDF Vectorized Logistic Regression 의 예측 정확도는 {0:.3f}'.format(accuracy_score(y_test ,pred)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "TF-IDF Vectorized Logistic Regression 의 예측 정확도는 0.692\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "FQTm5hroRlK6",
        "outputId": "df8c370b-ba90-44bc-d387-3db3a0f69d7c"
      },
      "source": [
        "from sklearn.model_selection import GridSearchCV\n",
        "\n",
        "# 최적 C 값 도출 튜닝 수행. CV는 3 Fold셋으로 설정. \n",
        "params = { 'C':[0.01, 0.1, 1, 5, 10]}\n",
        "grid_cv_lr = GridSearchCV(lr_clf ,param_grid=params , cv=3 , scoring='accuracy' , verbose=1 )\n",
        "grid_cv_lr.fit(X_train_tfidf_vect , y_train)\n",
        "print('Logistic Regression best C parameter :',grid_cv_lr.best_params_ )\n",
        "\n",
        "# 최적 C 값으로 학습된 grid_cv로 예측 수행하고 정확도 평가. \n",
        "pred = grid_cv_lr.predict(X_test_tfidf_vect)\n",
        "print('TF-IDF Vectorized Logistic Regression 의 예측 정확도는 {0:.3f}'.format(accuracy_score(y_test ,pred)))"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Fitting 3 folds for each of 5 candidates, totalling 15 fits\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
            "ERROR:root:Internal Python error in the inspect module.\n",
            "Below is the traceback from this internal error.\n",
            "\n",
            "ERROR:root:Internal Python error in the inspect module.\n",
            "Below is the traceback from this internal error.\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/IPython/core/interactiveshell.py\", line 2882, in run_code\n",
            "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
            "  File \"<ipython-input-10-b47517ccdf05>\", line 6, in <module>\n",
            "    grid_cv_lr.fit(X_train_tfidf_vect , y_train)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/sklearn/model_selection/_search.py\", line 710, in fit\n",
            "    self._run_search(evaluate_candidates)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/sklearn/model_selection/_search.py\", line 1151, in _run_search\n",
            "    evaluate_candidates(ParameterGrid(self.param_grid))\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/sklearn/model_selection/_search.py\", line 689, in evaluate_candidates\n",
            "    cv.split(X, y, groups)))\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/joblib/parallel.py\", line 1044, in __call__\n",
            "    while self.dispatch_one_batch(iterator):\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/joblib/parallel.py\", line 859, in dispatch_one_batch\n",
            "    self._dispatch(tasks)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/joblib/parallel.py\", line 777, in _dispatch\n",
            "    job = self._backend.apply_async(batch, callback=cb)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/joblib/_parallel_backends.py\", line 208, in apply_async\n",
            "    result = ImmediateResult(func)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/joblib/_parallel_backends.py\", line 572, in __init__\n",
            "    self.results = batch()\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/joblib/parallel.py\", line 263, in __call__\n",
            "    for func, args, kwargs in self.items]\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/joblib/parallel.py\", line 263, in <listcomp>\n",
            "    for func, args, kwargs in self.items]\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/sklearn/model_selection/_validation.py\", line 515, in _fit_and_score\n",
            "    estimator.fit(X_train, y_train, **fit_params)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py\", line 1601, in fit\n",
            "    for class_, warm_start_coef_ in zip(classes_, warm_start_coef))\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/joblib/parallel.py\", line 1041, in __call__\n",
            "    if self.dispatch_one_batch(iterator):\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/joblib/parallel.py\", line 859, in dispatch_one_batch\n",
            "    self._dispatch(tasks)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/joblib/parallel.py\", line 777, in _dispatch\n",
            "    job = self._backend.apply_async(batch, callback=cb)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/joblib/_parallel_backends.py\", line 208, in apply_async\n",
            "    result = ImmediateResult(func)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/joblib/_parallel_backends.py\", line 572, in __init__\n",
            "    self.results = batch()\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/joblib/parallel.py\", line 263, in __call__\n",
            "    for func, args, kwargs in self.items]\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/joblib/parallel.py\", line 263, in <listcomp>\n",
            "    for func, args, kwargs in self.items]\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py\", line 936, in _logistic_regression_path\n",
            "    options={\"iprint\": iprint, \"gtol\": tol, \"maxiter\": max_iter}\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/scipy/optimize/_minimize.py\", line 610, in minimize\n",
            "    callback=callback, **options)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/scipy/optimize/lbfgsb.py\", line 338, in _minimize_lbfgsb\n",
            "    isave, dsave, maxls)\n",
            "KeyboardInterrupt\n",
            "\n",
            "During handling of the above exception, another exception occurred:\n",
            "\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/IPython/core/interactiveshell.py\", line 1823, in showtraceback\n",
            "    stb = value._render_traceback_()\n",
            "AttributeError: 'KeyboardInterrupt' object has no attribute '_render_traceback_'\n",
            "\n",
            "During handling of the above exception, another exception occurred:\n",
            "\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/IPython/core/ultratb.py\", line 1132, in get_records\n",
            "    return _fixed_getinnerframes(etb, number_of_lines_of_context, tb_offset)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/IPython/core/ultratb.py\", line 313, in wrapped\n",
            "    return f(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/IPython/core/ultratb.py\", line 358, in _fixed_getinnerframes\n",
            "    records = fix_frame_records_filenames(inspect.getinnerframes(etb, context))\n",
            "  File \"/usr/lib/python3.7/inspect.py\", line 1502, in getinnerframes\n",
            "    frameinfo = (tb.tb_frame,) + getframeinfo(tb, context)\n",
            "  File \"/usr/lib/python3.7/inspect.py\", line 1460, in getframeinfo\n",
            "    filename = getsourcefile(frame) or getfile(frame)\n",
            "  File \"/usr/lib/python3.7/inspect.py\", line 696, in getsourcefile\n",
            "    if getattr(getmodule(object, filename), '__loader__', None) is not None:\n",
            "  File \"/usr/lib/python3.7/inspect.py\", line 739, in getmodule\n",
            "    f = getabsfile(module)\n",
            "  File \"/usr/lib/python3.7/inspect.py\", line 708, in getabsfile\n",
            "    _filename = getsourcefile(object) or getfile(object)\n",
            "  File \"/usr/lib/python3.7/inspect.py\", line 693, in getsourcefile\n",
            "    if os.path.exists(filename):\n",
            "  File \"/usr/lib/python3.7/genericpath.py\", line 19, in exists\n",
            "    os.stat(path)\n",
            "KeyboardInterrupt\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/IPython/core/interactiveshell.py\", line 2882, in run_code\n",
            "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
            "  File \"<ipython-input-10-b47517ccdf05>\", line 6, in <module>\n",
            "    grid_cv_lr.fit(X_train_tfidf_vect , y_train)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/sklearn/model_selection/_search.py\", line 710, in fit\n",
            "    self._run_search(evaluate_candidates)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/sklearn/model_selection/_search.py\", line 1151, in _run_search\n",
            "    evaluate_candidates(ParameterGrid(self.param_grid))\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/sklearn/model_selection/_search.py\", line 689, in evaluate_candidates\n",
            "    cv.split(X, y, groups)))\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/joblib/parallel.py\", line 1044, in __call__\n",
            "    while self.dispatch_one_batch(iterator):\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/joblib/parallel.py\", line 859, in dispatch_one_batch\n",
            "    self._dispatch(tasks)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/joblib/parallel.py\", line 777, in _dispatch\n",
            "    job = self._backend.apply_async(batch, callback=cb)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/joblib/_parallel_backends.py\", line 208, in apply_async\n",
            "    result = ImmediateResult(func)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/joblib/_parallel_backends.py\", line 572, in __init__\n",
            "    self.results = batch()\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/joblib/parallel.py\", line 263, in __call__\n",
            "    for func, args, kwargs in self.items]\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/joblib/parallel.py\", line 263, in <listcomp>\n",
            "    for func, args, kwargs in self.items]\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/sklearn/model_selection/_validation.py\", line 515, in _fit_and_score\n",
            "    estimator.fit(X_train, y_train, **fit_params)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py\", line 1601, in fit\n",
            "    for class_, warm_start_coef_ in zip(classes_, warm_start_coef))\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/joblib/parallel.py\", line 1041, in __call__\n",
            "    if self.dispatch_one_batch(iterator):\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/joblib/parallel.py\", line 859, in dispatch_one_batch\n",
            "    self._dispatch(tasks)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/joblib/parallel.py\", line 777, in _dispatch\n",
            "    job = self._backend.apply_async(batch, callback=cb)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/joblib/_parallel_backends.py\", line 208, in apply_async\n",
            "    result = ImmediateResult(func)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/joblib/_parallel_backends.py\", line 572, in __init__\n",
            "    self.results = batch()\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/joblib/parallel.py\", line 263, in __call__\n",
            "    for func, args, kwargs in self.items]\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/joblib/parallel.py\", line 263, in <listcomp>\n",
            "    for func, args, kwargs in self.items]\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py\", line 936, in _logistic_regression_path\n",
            "    options={\"iprint\": iprint, \"gtol\": tol, \"maxiter\": max_iter}\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/scipy/optimize/_minimize.py\", line 610, in minimize\n",
            "    callback=callback, **options)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/scipy/optimize/lbfgsb.py\", line 338, in _minimize_lbfgsb\n",
            "    isave, dsave, maxls)\n",
            "KeyboardInterrupt\n",
            "\n",
            "During handling of the above exception, another exception occurred:\n",
            "\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/IPython/core/interactiveshell.py\", line 1823, in showtraceback\n",
            "    stb = value._render_traceback_()\n",
            "AttributeError: 'KeyboardInterrupt' object has no attribute '_render_traceback_'\n",
            "\n",
            "During handling of the above exception, another exception occurred:\n",
            "\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/IPython/core/ultratb.py\", line 1132, in get_records\n",
            "    return _fixed_getinnerframes(etb, number_of_lines_of_context, tb_offset)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/IPython/core/ultratb.py\", line 313, in wrapped\n",
            "    return f(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/IPython/core/ultratb.py\", line 358, in _fixed_getinnerframes\n",
            "    records = fix_frame_records_filenames(inspect.getinnerframes(etb, context))\n",
            "  File \"/usr/lib/python3.7/inspect.py\", line 1502, in getinnerframes\n",
            "    frameinfo = (tb.tb_frame,) + getframeinfo(tb, context)\n",
            "  File \"/usr/lib/python3.7/inspect.py\", line 1460, in getframeinfo\n",
            "    filename = getsourcefile(frame) or getfile(frame)\n",
            "  File \"/usr/lib/python3.7/inspect.py\", line 696, in getsourcefile\n",
            "    if getattr(getmodule(object, filename), '__loader__', None) is not None:\n",
            "  File \"/usr/lib/python3.7/inspect.py\", line 739, in getmodule\n",
            "    f = getabsfile(module)\n",
            "  File \"/usr/lib/python3.7/inspect.py\", line 708, in getabsfile\n",
            "    _filename = getsourcefile(object) or getfile(object)\n",
            "  File \"/usr/lib/python3.7/inspect.py\", line 693, in getsourcefile\n",
            "    if os.path.exists(filename):\n",
            "  File \"/usr/lib/python3.7/genericpath.py\", line 19, in exists\n",
            "    os.stat(path)\n",
            "KeyboardInterrupt\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jjcueZGeRool"
      },
      "source": [
        "from sklearn.pipeline import Pipeline\n",
        "\n",
        "# TfidfVectorizer 객체를 tfidf_vect 객체명으로, LogisticRegression객체를 lr_clf 객체명으로 생성하는 Pipeline생성\n",
        "pipeline = Pipeline([\n",
        "    ('tfidf_vect', TfidfVectorizer(stop_words='english', ngram_range=(1,2), max_df=300)),\n",
        "    ('lr_clf', LogisticRegression(C=10))\n",
        "])\n",
        "\n",
        "# 별도의 TfidfVectorizer객체의 fit_transform( )과 LogisticRegression의 fit(), predict( )가 필요 없음. \n",
        "# pipeline의 fit( ) 과 predict( ) 만으로 한꺼번에 Feature Vectorization과 ML 학습/예측이 가능. \n",
        "pipeline.fit(X_train, y_train)\n",
        "pred = pipeline.predict(X_test)\n",
        "print('Pipeline을 통한 Logistic Regression 의 예측 정확도는 {0:.3f}'.format(accuracy_score(y_test ,pred)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M9WocCsqRtNC"
      },
      "source": [
        "from sklearn.pipeline import Pipeline\n",
        "\n",
        "pipeline = Pipeline([\n",
        "    ('tfidf_vect', TfidfVectorizer(stop_words='english')),\n",
        "    ('lr_clf', LogisticRegression())\n",
        "])\n",
        "\n",
        "# Pipeline에 기술된 각각의 객체 변수에 언더바(_)2개를 연달아 붙여 GridSearchCV에 사용될 \n",
        "# 파라미터/하이퍼 파라미터 이름과 값을 설정. . \n",
        "params = { 'tfidf_vect__ngram_range': [(1,1), (1,2), (1,3)],\n",
        "           'tfidf_vect__max_df': [100, 300, 700],\n",
        "           'lr_clf__C': [1,5,10]\n",
        "}\n",
        "\n",
        "# GridSearchCV의 생성자에 Estimator가 아닌 Pipeline 객체 입력\n",
        "grid_cv_pipe = GridSearchCV(pipeline, param_grid=params, cv=3 , scoring='accuracy',verbose=1)\n",
        "grid_cv_pipe.fit(X_train , y_train)\n",
        "print(grid_cv_pipe.best_params_ , grid_cv_pipe.best_score_)\n",
        "\n",
        "pred = grid_cv_pipe.predict(X_test)\n",
        "print('Pipeline을 통한 Logistic Regression 의 예측 정확도는 {0:.3f}'.format(accuracy_score(y_test ,pred)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wgbaz4TzRuH8"
      },
      "source": [
        "#497"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HfpEnNm6Rxru",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 142
        },
        "outputId": "a4e49247-4214-448d-fb4f-ab1c2ace909e"
      },
      "source": [
        "import pandas as pd\n",
        "\n",
        "review_df = pd.read_csv('/content/labeledTrainData.tsv', header= 0, sep='\\t', quoting=3)\n",
        "review_df.head(3)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>sentiment</th>\n",
              "      <th>review</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>\"5814_8\"</td>\n",
              "      <td>1</td>\n",
              "      <td>\"With all this stuff going down at the moment ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>\"2381_9\"</td>\n",
              "      <td>1</td>\n",
              "      <td>\"\\\"The Classic War of the Worlds\\\" by Timothy ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>\"7759_3\"</td>\n",
              "      <td>0</td>\n",
              "      <td>\"The film starts with a manager (Nicholas Bell...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "         id  sentiment                                             review\n",
              "0  \"5814_8\"          1  \"With all this stuff going down at the moment ...\n",
              "1  \"2381_9\"          1  \"\\\"The Classic War of the Worlds\\\" by Timothy ...\n",
              "2  \"7759_3\"          0  \"The film starts with a manager (Nicholas Bell..."
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WBLrt3PSR4Y4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0f45b81a-dd30-4235-ec7d-db10006912ff"
      },
      "source": [
        "print(review_df['review'][0])"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\"With all this stuff going down at the moment with MJ i've started listening to his music, watching the odd documentary here and there, watched The Wiz and watched Moonwalker again. Maybe i just want to get a certain insight into this guy who i thought was really cool in the eighties just to maybe make up my mind whether he is guilty or innocent. Moonwalker is part biography, part feature film which i remember going to see at the cinema when it was originally released. Some of it has subtle messages about MJ's feeling towards the press and also the obvious message of drugs are bad m'kay.<br /><br />Visually impressive but of course this is all about Michael Jackson so unless you remotely like MJ in anyway then you are going to hate this and find it boring. Some may call MJ an egotist for consenting to the making of this movie BUT MJ and most of his fans would say that he made it for the fans which if true is really nice of him.<br /><br />The actual feature film bit when it finally starts is only on for 20 minutes or so excluding the Smooth Criminal sequence and Joe Pesci is convincing as a psychopathic all powerful drug lord. Why he wants MJ dead so bad is beyond me. Because MJ overheard his plans? Nah, Joe Pesci's character ranted that he wanted people to know it is he who is supplying drugs etc so i dunno, maybe he just hates MJ's music.<br /><br />Lots of cool things in this like MJ turning into a car and a robot and the whole Speed Demon sequence. Also, the director must have had the patience of a saint when it came to filming the kiddy Bad sequence as usually directors hate working with one kid let alone a whole bunch of them performing a complex dance scene.<br /><br />Bottom line, this movie is for people who like MJ on one level or another (which i think is most people). If not, then stay away. It does try and give off a wholesome message and ironically MJ's bestest buddy in this movie is a girl! Michael Jackson is truly one of the most talented people ever to grace this planet but is he guilty? Well, with all the attention i've gave this subject....hmmm well i don't know because people can be different behind closed doors, i know this for a fact. He is either an extremely nice but stupid guy or one of the most sickest liars. I hope he is not the latter.\"\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pKJDBjbgSADF"
      },
      "source": [
        "import re\n",
        "\n",
        "review_df['review'] = review_df['review'].str.replace('<br />',' ')\n",
        "# <br> html 태그를 replace 함수로 공백으로 변환\n",
        "\n",
        "review_df['review'] = review_df['review'].apply( lambda x : re.sub(\"[^a-zA-z]\",\" \",x))\n",
        "# 파이썬의 정규 표현식 모듈인 re를 이용해 영어 문자열이 아닌 문자는 모두 공백으로 변환"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PqfFn3fYR9us",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0a878595-5e58-4ef5-bbce-068218b2a93a"
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "class_df = review_df['sentiment']\n",
        "feature_df = review_df.drop(['id','sentiment'],axis=1,inplace=False)\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(feature_df, class_df,test_size=0.3, random_state=156)\n",
        "\n",
        "X_train.shape, X_test.shape"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((9746, 1), (4177, 1))"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b-9-2mKySE0T",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "579a2a39-4aab-445d-8264-d03f7b548779"
      },
      "source": [
        "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
        "from sklearn.pipeline import Pipeline\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.metrics import accuracy_score, roc_auc_score\n",
        "\n",
        "# 스톱워드는 english, fitering, ngram은 (1,2)로 설정해 countvectorization 수행\n",
        "# logistic regression의 c는 10으로 설정\n",
        "\n",
        "pipeline = Pipeline([\n",
        "    ('cnt_vect', CountVectorizer(stop_words='english', ngram_range=(1,2))),\n",
        "    ('lr_clf', LogisticRegression(C=10))\n",
        "])\n",
        "\n",
        "# pipeline 객체를 이용해 fit(), predict()로 학습/예측 수행. predict_proba()는 roc_auc 때문에 수행.\n",
        "pipeline.fit(X_train['review'],y_train)\n",
        "pred = pipeline.predict(X_test['review'])\n",
        "pred_probs = pipeline.predict_proba(X_test['review'])[:,1]\n",
        "\n",
        "print('예측정확도는 {0:.4f}, Roc-auc는 {1:.4f}'.format(accuracy_score(y_test,pred),roc_auc_score(y_test,pred_probs)))"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "예측정확도는 0.8760, Roc-auc는 0.9377\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bivHdM-hSGx4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1c0117e9-1e19-4ec6-fc54-abb5cbbfb7d1"
      },
      "source": [
        "# wordnet synset과 sentiwordnet sentisynset 클래스의 이해\n",
        "\n",
        "import nltk\n",
        "nltk.download('all')"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading collection 'all'\n",
            "[nltk_data]    | \n",
            "[nltk_data]    | Downloading package abc to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/abc.zip.\n",
            "[nltk_data]    | Downloading package alpino to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/alpino.zip.\n",
            "[nltk_data]    | Downloading package biocreative_ppi to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/biocreative_ppi.zip.\n",
            "[nltk_data]    | Downloading package brown to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/brown.zip.\n",
            "[nltk_data]    | Downloading package brown_tei to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/brown_tei.zip.\n",
            "[nltk_data]    | Downloading package cess_cat to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/cess_cat.zip.\n",
            "[nltk_data]    | Downloading package cess_esp to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/cess_esp.zip.\n",
            "[nltk_data]    | Downloading package chat80 to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/chat80.zip.\n",
            "[nltk_data]    | Downloading package city_database to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/city_database.zip.\n",
            "[nltk_data]    | Downloading package cmudict to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/cmudict.zip.\n",
            "[nltk_data]    | Downloading package comparative_sentences to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/comparative_sentences.zip.\n",
            "[nltk_data]    | Downloading package comtrans to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package conll2000 to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/conll2000.zip.\n",
            "[nltk_data]    | Downloading package conll2002 to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/conll2002.zip.\n",
            "[nltk_data]    | Downloading package conll2007 to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package crubadan to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/crubadan.zip.\n",
            "[nltk_data]    | Downloading package dependency_treebank to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/dependency_treebank.zip.\n",
            "[nltk_data]    | Downloading package dolch to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/dolch.zip.\n",
            "[nltk_data]    | Downloading package europarl_raw to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/europarl_raw.zip.\n",
            "[nltk_data]    | Downloading package floresta to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/floresta.zip.\n",
            "[nltk_data]    | Downloading package framenet_v15 to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/framenet_v15.zip.\n",
            "[nltk_data]    | Downloading package framenet_v17 to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/framenet_v17.zip.\n",
            "[nltk_data]    | Downloading package gazetteers to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/gazetteers.zip.\n",
            "[nltk_data]    | Downloading package genesis to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/genesis.zip.\n",
            "[nltk_data]    | Downloading package gutenberg to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/gutenberg.zip.\n",
            "[nltk_data]    | Downloading package ieer to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/ieer.zip.\n",
            "[nltk_data]    | Downloading package inaugural to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/inaugural.zip.\n",
            "[nltk_data]    | Downloading package indian to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/indian.zip.\n",
            "[nltk_data]    | Downloading package jeita to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package kimmo to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/kimmo.zip.\n",
            "[nltk_data]    | Downloading package knbc to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package lin_thesaurus to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/lin_thesaurus.zip.\n",
            "[nltk_data]    | Downloading package mac_morpho to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/mac_morpho.zip.\n",
            "[nltk_data]    | Downloading package machado to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package masc_tagged to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package moses_sample to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping models/moses_sample.zip.\n",
            "[nltk_data]    | Downloading package movie_reviews to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/movie_reviews.zip.\n",
            "[nltk_data]    | Downloading package names to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/names.zip.\n",
            "[nltk_data]    | Downloading package nombank.1.0 to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package nps_chat to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/nps_chat.zip.\n",
            "[nltk_data]    | Downloading package omw to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/omw.zip.\n",
            "[nltk_data]    | Downloading package opinion_lexicon to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/opinion_lexicon.zip.\n",
            "[nltk_data]    | Downloading package paradigms to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/paradigms.zip.\n",
            "[nltk_data]    | Downloading package pil to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/pil.zip.\n",
            "[nltk_data]    | Downloading package pl196x to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/pl196x.zip.\n",
            "[nltk_data]    | Downloading package ppattach to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/ppattach.zip.\n",
            "[nltk_data]    | Downloading package problem_reports to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/problem_reports.zip.\n",
            "[nltk_data]    | Downloading package propbank to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package ptb to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/ptb.zip.\n",
            "[nltk_data]    | Downloading package product_reviews_1 to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/product_reviews_1.zip.\n",
            "[nltk_data]    | Downloading package product_reviews_2 to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/product_reviews_2.zip.\n",
            "[nltk_data]    | Downloading package pros_cons to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/pros_cons.zip.\n",
            "[nltk_data]    | Downloading package qc to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/qc.zip.\n",
            "[nltk_data]    | Downloading package reuters to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package rte to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/rte.zip.\n",
            "[nltk_data]    | Downloading package semcor to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package senseval to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/senseval.zip.\n",
            "[nltk_data]    | Downloading package sentiwordnet to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/sentiwordnet.zip.\n",
            "[nltk_data]    | Downloading package sentence_polarity to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/sentence_polarity.zip.\n",
            "[nltk_data]    | Downloading package shakespeare to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/shakespeare.zip.\n",
            "[nltk_data]    | Downloading package sinica_treebank to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/sinica_treebank.zip.\n",
            "[nltk_data]    | Downloading package smultron to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/smultron.zip.\n",
            "[nltk_data]    | Downloading package state_union to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/state_union.zip.\n",
            "[nltk_data]    | Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/stopwords.zip.\n",
            "[nltk_data]    | Downloading package subjectivity to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/subjectivity.zip.\n",
            "[nltk_data]    | Downloading package swadesh to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/swadesh.zip.\n",
            "[nltk_data]    | Downloading package switchboard to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/switchboard.zip.\n",
            "[nltk_data]    | Downloading package timit to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/timit.zip.\n",
            "[nltk_data]    | Downloading package toolbox to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/toolbox.zip.\n",
            "[nltk_data]    | Downloading package treebank to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/treebank.zip.\n",
            "[nltk_data]    | Downloading package twitter_samples to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/twitter_samples.zip.\n",
            "[nltk_data]    | Downloading package udhr to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/udhr.zip.\n",
            "[nltk_data]    | Downloading package udhr2 to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/udhr2.zip.\n",
            "[nltk_data]    | Downloading package unicode_samples to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/unicode_samples.zip.\n",
            "[nltk_data]    | Downloading package universal_treebanks_v20 to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    | Downloading package verbnet to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/verbnet.zip.\n",
            "[nltk_data]    | Downloading package verbnet3 to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/verbnet3.zip.\n",
            "[nltk_data]    | Downloading package webtext to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/webtext.zip.\n",
            "[nltk_data]    | Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/wordnet.zip.\n",
            "[nltk_data]    | Downloading package wordnet_ic to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/wordnet_ic.zip.\n",
            "[nltk_data]    | Downloading package words to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/words.zip.\n",
            "[nltk_data]    | Downloading package ycoe to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/ycoe.zip.\n",
            "[nltk_data]    | Downloading package rslp to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping stemmers/rslp.zip.\n",
            "[nltk_data]    | Downloading package maxent_treebank_pos_tagger to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping taggers/maxent_treebank_pos_tagger.zip.\n",
            "[nltk_data]    | Downloading package universal_tagset to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping taggers/universal_tagset.zip.\n",
            "[nltk_data]    | Downloading package maxent_ne_chunker to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping chunkers/maxent_ne_chunker.zip.\n",
            "[nltk_data]    | Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data]    | Downloading package book_grammars to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping grammars/book_grammars.zip.\n",
            "[nltk_data]    | Downloading package sample_grammars to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping grammars/sample_grammars.zip.\n",
            "[nltk_data]    | Downloading package spanish_grammars to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping grammars/spanish_grammars.zip.\n",
            "[nltk_data]    | Downloading package basque_grammars to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping grammars/basque_grammars.zip.\n",
            "[nltk_data]    | Downloading package large_grammars to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping grammars/large_grammars.zip.\n",
            "[nltk_data]    | Downloading package tagsets to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping help/tagsets.zip.\n",
            "[nltk_data]    | Downloading package snowball_data to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    | Downloading package bllip_wsj_no_aux to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping models/bllip_wsj_no_aux.zip.\n",
            "[nltk_data]    | Downloading package word2vec_sample to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping models/word2vec_sample.zip.\n",
            "[nltk_data]    | Downloading package panlex_swadesh to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    | Downloading package mte_teip5 to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/mte_teip5.zip.\n",
            "[nltk_data]    | Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping taggers/averaged_perceptron_tagger.zip.\n",
            "[nltk_data]    | Downloading package averaged_perceptron_tagger_ru to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping\n",
            "[nltk_data]    |       taggers/averaged_perceptron_tagger_ru.zip.\n",
            "[nltk_data]    | Downloading package perluniprops to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping misc/perluniprops.zip.\n",
            "[nltk_data]    | Downloading package nonbreaking_prefixes to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/nonbreaking_prefixes.zip.\n",
            "[nltk_data]    | Downloading package vader_lexicon to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    | Downloading package porter_test to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping stemmers/porter_test.zip.\n",
            "[nltk_data]    | Downloading package wmt15_eval to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping models/wmt15_eval.zip.\n",
            "[nltk_data]    | Downloading package mwa_ppdb to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping misc/mwa_ppdb.zip.\n",
            "[nltk_data]    | \n",
            "[nltk_data]  Done downloading collection all\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MoJQWGcNSJvI",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2323d8c9-af4d-4b24-c85a-ebd6aeb2909d"
      },
      "source": [
        "from nltk.corpus import wordnet as wn\n",
        "\n",
        "term = 'present'\n",
        "\n",
        "# present라는 단어로 wordnet의 synsets 생성.\n",
        "synsets = wn.synsets(term)\n",
        "print('synsets() 반환 type :', type(synsets))\n",
        "print('synsets() 반환 값 개수:', len(synsets))\n",
        "print('synsets() 반환 값 :', synsets)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "synsets() 반환 type : <class 'list'>\n",
            "synsets() 반환 값 개수: 18\n",
            "synsets() 반환 값 : [Synset('present.n.01'), Synset('present.n.02'), Synset('present.n.03'), Synset('show.v.01'), Synset('present.v.02'), Synset('stage.v.01'), Synset('present.v.04'), Synset('present.v.05'), Synset('award.v.01'), Synset('give.v.08'), Synset('deliver.v.01'), Synset('introduce.v.01'), Synset('portray.v.04'), Synset('confront.v.03'), Synset('present.v.12'), Synset('salute.v.06'), Synset('present.a.01'), Synset('present.a.02')]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N3dZxA3TSLaM",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f0badc14-db86-4482-c311-426a46f4f3ac"
      },
      "source": [
        "for synset in synsets :\n",
        "    print('#### synset name : ', synset.name(),'####')\n",
        "    print('POS :', synset.lexname())\n",
        "    print('Definition:', synset.definition())\n",
        "    print('Lemmas:', synset.lemma_names())"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "#### synset name :  present.n.01 ####\n",
            "POS : noun.time\n",
            "Definition: the period of time that is happening now; any continuous stretch of time including the moment of speech\n",
            "Lemmas: ['present', 'nowadays']\n",
            "#### synset name :  present.n.02 ####\n",
            "POS : noun.possession\n",
            "Definition: something presented as a gift\n",
            "Lemmas: ['present']\n",
            "#### synset name :  present.n.03 ####\n",
            "POS : noun.communication\n",
            "Definition: a verb tense that expresses actions or states at the time of speaking\n",
            "Lemmas: ['present', 'present_tense']\n",
            "#### synset name :  show.v.01 ####\n",
            "POS : verb.perception\n",
            "Definition: give an exhibition of to an interested audience\n",
            "Lemmas: ['show', 'demo', 'exhibit', 'present', 'demonstrate']\n",
            "#### synset name :  present.v.02 ####\n",
            "POS : verb.communication\n",
            "Definition: bring forward and present to the mind\n",
            "Lemmas: ['present', 'represent', 'lay_out']\n",
            "#### synset name :  stage.v.01 ####\n",
            "POS : verb.creation\n",
            "Definition: perform (a play), especially on a stage\n",
            "Lemmas: ['stage', 'present', 'represent']\n",
            "#### synset name :  present.v.04 ####\n",
            "POS : verb.possession\n",
            "Definition: hand over formally\n",
            "Lemmas: ['present', 'submit']\n",
            "#### synset name :  present.v.05 ####\n",
            "POS : verb.stative\n",
            "Definition: introduce\n",
            "Lemmas: ['present', 'pose']\n",
            "#### synset name :  award.v.01 ####\n",
            "POS : verb.possession\n",
            "Definition: give, especially as an honor or reward\n",
            "Lemmas: ['award', 'present']\n",
            "#### synset name :  give.v.08 ####\n",
            "POS : verb.possession\n",
            "Definition: give as a present; make a gift of\n",
            "Lemmas: ['give', 'gift', 'present']\n",
            "#### synset name :  deliver.v.01 ####\n",
            "POS : verb.communication\n",
            "Definition: deliver (a speech, oration, or idea)\n",
            "Lemmas: ['deliver', 'present']\n",
            "#### synset name :  introduce.v.01 ####\n",
            "POS : verb.communication\n",
            "Definition: cause to come to know personally\n",
            "Lemmas: ['introduce', 'present', 'acquaint']\n",
            "#### synset name :  portray.v.04 ####\n",
            "POS : verb.creation\n",
            "Definition: represent abstractly, for example in a painting, drawing, or sculpture\n",
            "Lemmas: ['portray', 'present']\n",
            "#### synset name :  confront.v.03 ####\n",
            "POS : verb.communication\n",
            "Definition: present somebody with something, usually to accuse or criticize\n",
            "Lemmas: ['confront', 'face', 'present']\n",
            "#### synset name :  present.v.12 ####\n",
            "POS : verb.communication\n",
            "Definition: formally present a debutante, a representative of a country, etc.\n",
            "Lemmas: ['present']\n",
            "#### synset name :  salute.v.06 ####\n",
            "POS : verb.communication\n",
            "Definition: recognize with a gesture prescribed by a military regulation; assume a prescribed position\n",
            "Lemmas: ['salute', 'present']\n",
            "#### synset name :  present.a.01 ####\n",
            "POS : adj.all\n",
            "Definition: temporal sense; intermediate between past and future; now existing or happening or in consideration\n",
            "Lemmas: ['present']\n",
            "#### synset name :  present.a.02 ####\n",
            "POS : adj.all\n",
            "Definition: being or existing in a specified place\n",
            "Lemmas: ['present']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "adj9PAlqSNn0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "d6175870-8497-40ef-d922-e0b438a03673"
      },
      "source": [
        "\n",
        "# synset 객체를 단어별로 생성\n",
        "tree = wn.synset('tree.n.01')\n",
        "lion = wn.synset('lion.n.01')\n",
        "tiger = wn.synset('tiger.n.02')\n",
        "cat = wn.synset('cat.n.01')\n",
        "dog = wn.synset('dog.n.01')\n",
        "\n",
        "entities = [tree, lion, tiger, cat, dog]\n",
        "similarities = []\n",
        "entity_names = [entity.name().split('.')[0] for entity in entities]\n",
        "\n",
        "# 단어별 synset을 반복하며 다른 단어의 synset과 유사도 측정\n",
        "for entity in entities:\n",
        "    similarity = [round(entity.path_similarity(compared_entity),2)\n",
        "                 for compared_entity in entities]\n",
        "    similarities.append(similarity)\n",
        "    \n",
        "#개별 단어별 synset과 다른 단어의 synset과의 유사도를 dataframe형태로 저장\n",
        "similarity_df = pd.DataFrame(similarities, columns = entity_names, index = entity_names )\n",
        "similarity_df"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tree</th>\n",
              "      <th>lion</th>\n",
              "      <th>tiger</th>\n",
              "      <th>cat</th>\n",
              "      <th>dog</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>tree</th>\n",
              "      <td>1.00</td>\n",
              "      <td>0.07</td>\n",
              "      <td>0.07</td>\n",
              "      <td>0.08</td>\n",
              "      <td>0.12</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>lion</th>\n",
              "      <td>0.07</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.33</td>\n",
              "      <td>0.25</td>\n",
              "      <td>0.17</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>tiger</th>\n",
              "      <td>0.07</td>\n",
              "      <td>0.33</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.25</td>\n",
              "      <td>0.17</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>cat</th>\n",
              "      <td>0.08</td>\n",
              "      <td>0.25</td>\n",
              "      <td>0.25</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.20</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>dog</th>\n",
              "      <td>0.12</td>\n",
              "      <td>0.17</td>\n",
              "      <td>0.17</td>\n",
              "      <td>0.20</td>\n",
              "      <td>1.00</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       tree  lion  tiger   cat   dog\n",
              "tree   1.00  0.07   0.07  0.08  0.12\n",
              "lion   0.07  1.00   0.33  0.25  0.17\n",
              "tiger  0.07  0.33   1.00  0.25  0.17\n",
              "cat    0.08  0.25   0.25  1.00  0.20\n",
              "dog    0.12  0.17   0.17  0.20  1.00"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g0FxwlRYSPj7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b5f86f4b-55f9-440b-afe8-a18f18b5046f"
      },
      "source": [
        "import nltk\n",
        "from nltk.corpus import sentiwordnet as swn\n",
        "\n",
        "senti_synsets = list(swn.senti_synsets('slow'))\n",
        "print('senti_synsets() 반환 type :', type(senti_synsets))\n",
        "print('senti_synsets() 반환 값 개수:', len(senti_synsets))\n",
        "print('senti_synsets() 반환 값 :', senti_synsets)"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "senti_synsets() 반환 type : <class 'list'>\n",
            "senti_synsets() 반환 값 개수: 11\n",
            "senti_synsets() 반환 값 : [SentiSynset('decelerate.v.01'), SentiSynset('slow.v.02'), SentiSynset('slow.v.03'), SentiSynset('slow.a.01'), SentiSynset('slow.a.02'), SentiSynset('dense.s.04'), SentiSynset('slow.a.04'), SentiSynset('boring.s.01'), SentiSynset('dull.s.08'), SentiSynset('slowly.r.01'), SentiSynset('behind.r.03')]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rNkGmCWzSSHv",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8a221bf8-9026-4d46-8736-1cc1e5feaaa4"
      },
      "source": [
        "import nltk\n",
        "from nltk.corpus import sentiwordnet as swn\n",
        "\n",
        "father = swn.senti_synset('father.n.01')\n",
        "print('father 긍정감성 지수: ', father.pos_score())\n",
        "print('father 부정감성 지수: ', father.neg_score())\n",
        "print('father 객관성 지수: ', father.obj_score())\n",
        "print('\\n')\n",
        "\n",
        "fabulous = swn.senti_synset('fabulous.a.01')\n",
        "print('fabulous 긍정감성 지수: ', fabulous.pos_score() )\n",
        "print('fabulous 부정감성 지수: ', fabulous.neg_score())"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "father 긍정감성 지수:  0.0\n",
            "father 부정감성 지수:  0.0\n",
            "father 객관성 지수:  1.0\n",
            "\n",
            "\n",
            "fabulous 긍정감성 지수:  0.875\n",
            "fabulous 부정감성 지수:  0.125\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cazhaoTvST4p"
      },
      "source": [
        "from nltk.corpus import wordnet as wn\n",
        "\n",
        "# 간단한 NLTK PennTreebank Tag를 기반으로 WordNet 기반의 품사 Tag로 변환\n",
        "def penn_to_wn(tag):\n",
        "    if tag.startswith('J'):\n",
        "        return wn.ADJ\n",
        "    elif tag.startswith('N'):\n",
        "        return wn.NOUN\n",
        "    elif tag.startswith('R'):\n",
        "        return wn.ADV\n",
        "    elif tag.startswith('V'):\n",
        "        return wn.VERB"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MK_IyIxPSXew"
      },
      "source": [
        "from nltk.stem import WordNetLemmatizer\n",
        "from nltk.corpus import sentiwordnet as swn\n",
        "from nltk import sent_tokenize, word_tokenize, pos_tag\n",
        "\n",
        "def swn_polarity(text):\n",
        "    # 감성 지수 초기화 \n",
        "    sentiment = 0.0\n",
        "    tokens_count = 0\n",
        "    \n",
        "    lemmatizer = WordNetLemmatizer()\n",
        "    raw_sentences = sent_tokenize(text)\n",
        "    # 분해된 문장별로 단어 토큰 -> 품사 태깅 후에 SentiSynset 생성 -> 감성 지수 합산 \n",
        "    for raw_sentence in raw_sentences:\n",
        "        # NTLK 기반의 품사 태깅 문장 추출  \n",
        "        tagged_sentence = pos_tag(word_tokenize(raw_sentence))\n",
        "        for word , tag in tagged_sentence:\n",
        "            \n",
        "            # WordNet 기반 품사 태깅과 어근 추출\n",
        "            wn_tag = penn_to_wn(tag)\n",
        "            if wn_tag not in (wn.NOUN , wn.ADJ, wn.ADV):\n",
        "                continue                   \n",
        "            lemma = lemmatizer.lemmatize(word, pos=wn_tag)\n",
        "            if not lemma:\n",
        "                continue\n",
        "            # 어근을 추출한 단어와 WordNet 기반 품사 태깅을 입력해 Synset 객체를 생성. \n",
        "            synsets = wn.synsets(lemma , pos=wn_tag)\n",
        "            if not synsets:\n",
        "                continue\n",
        "            # sentiwordnet의 감성 단어 분석으로 감성 synset 추출\n",
        "            # 모든 단어에 대해 긍정 감성 지수는 +로 부정 감성 지수는 -로 합산해 감성 지수 계산. \n",
        "            synset = synsets[0]\n",
        "            swn_synset = swn.senti_synset(synset.name())\n",
        "            sentiment += (swn_synset.pos_score() - swn_synset.neg_score())           \n",
        "            tokens_count += 1\n",
        "    \n",
        "    if not tokens_count:\n",
        "        return 0\n",
        "    \n",
        "    # 총 score가 0 이상일 경우 긍정(Positive) 1, 그렇지 않을 경우 부정(Negative) 0 반환\n",
        "    if sentiment >= 0 :\n",
        "        return 1\n",
        "    \n",
        "    return 0"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Uh8s7-v_SccU"
      },
      "source": [
        "review_df['preds'] = review_df['review'].apply(lambda x: swn_polarity(x))\n",
        "y_target = review_df['sentiment'].values\n",
        "preds = review_df['preds'].values"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9ojaCpibSebO",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f14b21d7-aa8a-4c79-8f10-64f5c171c33c"
      },
      "source": [
        "from sklearn.metrics import accuracy_score, confusion_matrix, precision_score\n",
        "from sklearn.metrics import recall_score, f1_score, roc_auc_score\n",
        "import numpy as np\n",
        "\n",
        "print(confusion_matrix(y_target, preds))\n",
        "print(\"정확도:\", np.round(accuracy_score(y_target,preds),4))\n",
        "print(\"정밀도:\", np.round(precision_score(y_target,preds),4))\n",
        "print(\"재현율:\", np.round(recall_score(y_target,preds),4))"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[4256 2674]\n",
            " [2060 4933]]\n",
            "정확도: 0.66\n",
            "정밀도: 0.6485\n",
            "재현율: 0.7054\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q-DOWNAlSgL6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8c27ee40-7bbe-41c8-fa1b-12051b2d7125"
      },
      "source": [
        "# Vader\n",
        "\n",
        "from nltk.sentiment.vader import SentimentIntensityAnalyzer\n",
        "\n",
        "senti_analyzer = SentimentIntensityAnalyzer()\n",
        "senti_scores = senti_analyzer.polarity_scores(review_df['review'][0])\n",
        "print(senti_scores)"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'neg': 0.13, 'neu': 0.743, 'pos': 0.127, 'compound': -0.7943}\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/nltk/twitter/__init__.py:20: UserWarning: The twython library has not been installed. Some functionality from the twitter package will not be available.\n",
            "  warnings.warn(\"The twython library has not been installed. \"\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iHA6quoWSiHf",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "33367eb5-a384-4ef7-952b-aeeec7be9e6c"
      },
      "source": [
        "def vader_polarity(review,threshold=0.1):\n",
        "    analyzer = SentimentIntensityAnalyzer()\n",
        "    scores = analyzer.polarity_scores(review)\n",
        "    \n",
        "    # compound 값에 기반하여 threshold 입력값보다 크면 1, 그렇지 않으면 0을 반환 \n",
        "    agg_score = scores['compound']\n",
        "    final_sentiment = 1 if agg_score >= threshold else 0\n",
        "    return final_sentiment\n",
        "\n",
        "# apply lambda 식을 이용하여 레코드별로 vader_polarity( )를 수행하고 결과를 'vader_preds'에 저장\n",
        "review_df['vader_preds'] = review_df['review'].apply( lambda x : vader_polarity(x, 0.1) )\n",
        "y_target = review_df['sentiment'].values\n",
        "vader_preds = review_df['vader_preds'].values\n",
        "\n",
        "print(confusion_matrix( y_target, vader_preds))\n",
        "print(\"정확도:\", np.round(accuracy_score(y_target , vader_preds),4))\n",
        "print(\"정밀도:\", np.round(precision_score(y_target , vader_preds),4))\n",
        "print(\"재현율:\", np.round(recall_score(y_target, vader_preds),4))"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[3749 3181]\n",
            " [ 992 6001]]\n",
            "정확도: 0.7003\n",
            "정밀도: 0.6536\n",
            "재현율: 0.8581\n"
          ]
        }
      ]
    }
  ]
}