Lab3_2_BERT.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Lab7.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyPBLLajiXKoMuzUi4kwxkZg",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "b860e5773ed143cca217926091115ffb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b11fcbae2fea4deb9ec816bdaed80d47",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_923652d7f0bd45b0b4e939fd1d0d2da2",
              "IPY_MODEL_6c2f7e7e165845e9989ef27a8b08946a"
            ]
          }
        },
        "b11fcbae2fea4deb9ec816bdaed80d47": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "923652d7f0bd45b0b4e939fd1d0d2da2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_40781fad917b48f28f6304bb09ff657b",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 249928,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 249928,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e824541e2a504e2ba09bf1ee99eaf334"
          }
        },
        "6c2f7e7e165845e9989ef27a8b08946a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2071310744c5496db08ee96b82071367",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 250k/250k [00:00&lt;00:00, 459kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_19c544aa05f746c6bbbfdfe846292a1b"
          }
        },
        "40781fad917b48f28f6304bb09ff657b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e824541e2a504e2ba09bf1ee99eaf334": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2071310744c5496db08ee96b82071367": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "19c544aa05f746c6bbbfdfe846292a1b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "daa1ab91c2bb45f9b026a2a1fb9f49c3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_47adb3c24cfa4844b901564b3a6f8d4b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e15d758bdb234e9b8005cfa2be085073",
              "IPY_MODEL_f9186b7f712c44019b5a29c22b2ca740"
            ]
          }
        },
        "47adb3c24cfa4844b901564b3a6f8d4b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e15d758bdb234e9b8005cfa2be085073": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2266331bf3d64fbdb0dc8da79530d4f7",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 49,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 49,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0de116d41931489b83c0105016ce7cec"
          }
        },
        "f9186b7f712c44019b5a29c22b2ca740": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d87e3e92744a4a638d4d4ca8c03aaa45",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 49.0/49.0 [00:00&lt;00:00, 289B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e971b5dc9f55400ca146ae4775d1b642"
          }
        },
        "2266331bf3d64fbdb0dc8da79530d4f7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0de116d41931489b83c0105016ce7cec": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d87e3e92744a4a638d4d4ca8c03aaa45": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e971b5dc9f55400ca146ae4775d1b642": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dd7a14e1b71049ec8e30a213da049ddf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d2a1150a2f4d4a6689702b5a67536333",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_99bac06470884a5e9c8e192465fa28f5",
              "IPY_MODEL_0c7b748dbafd40dab65fba80a1fdeb77"
            ]
          }
        },
        "d2a1150a2f4d4a6689702b5a67536333": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "99bac06470884a5e9c8e192465fa28f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9f0c4ef6f5de4ea1919b9a2e5d211f2c",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 619,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 619,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6a80ae9b74204ceb95e74cfc73b4c18f"
          }
        },
        "0c7b748dbafd40dab65fba80a1fdeb77": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_04d2881abf924ce0aa9a9516880472aa",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 619/619 [00:13&lt;00:00, 46.9B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0989be37fa3149a397a76ba7a8bd48af"
          }
        },
        "9f0c4ef6f5de4ea1919b9a2e5d211f2c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6a80ae9b74204ceb95e74cfc73b4c18f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "04d2881abf924ce0aa9a9516880472aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0989be37fa3149a397a76ba7a8bd48af": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d802ddf1d1de490fbdfee58af5966a49": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0b097dd8923d43ab951d986351aa40f9",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_904409fa2e7c4463a63e7583ab8726cd",
              "IPY_MODEL_3dd25ca47d6e4fed8846ff370e39693d"
            ]
          }
        },
        "0b097dd8923d43ab951d986351aa40f9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "904409fa2e7c4463a63e7583ab8726cd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_0e95fa4fd79e4a06bd7d9e64b305ef01",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 438218004,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 438218004,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7e88c1adb28343468ec4c95634b76784"
          }
        },
        "3dd25ca47d6e4fed8846ff370e39693d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_64d8c3a1d09a429483bf1776d97af3ec",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 438M/438M [00:11&lt;00:00, 36.9MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4ce0205c763f42c1980740cc44a5cb46"
          }
        },
        "0e95fa4fd79e4a06bd7d9e64b305ef01": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7e88c1adb28343468ec4c95634b76784": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "64d8c3a1d09a429483bf1776d97af3ec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4ce0205c763f42c1980740cc44a5cb46": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/illhyhl1111/AI_Expert_2021/blob/main/Lab7.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EidE4YTn28gP"
      },
      "source": [
        "Code from https://ratsgo.github.io/nlpbook/docs/language_model/tutorial/"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AanLMepm2z5V"
      },
      "source": [
        "# 패키지 설치\n",
        "pip 명령어로 의존성 있는 패키지를 설치"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_KCr-1QM2qiS"
      },
      "source": [
        "!pip install ratsnlp"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gs4GqkCd2wEl"
      },
      "source": [
        "## 토크나이저 초기화\n",
        "\n",
        "BERT(`kcbert-base`) 모델이 쓰는 토크나이저를 선언"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 116,
          "referenced_widgets": [
            "b860e5773ed143cca217926091115ffb",
            "b11fcbae2fea4deb9ec816bdaed80d47",
            "923652d7f0bd45b0b4e939fd1d0d2da2",
            "6c2f7e7e165845e9989ef27a8b08946a",
            "40781fad917b48f28f6304bb09ff657b",
            "e824541e2a504e2ba09bf1ee99eaf334",
            "2071310744c5496db08ee96b82071367",
            "19c544aa05f746c6bbbfdfe846292a1b",
            "daa1ab91c2bb45f9b026a2a1fb9f49c3",
            "47adb3c24cfa4844b901564b3a6f8d4b",
            "e15d758bdb234e9b8005cfa2be085073",
            "f9186b7f712c44019b5a29c22b2ca740",
            "2266331bf3d64fbdb0dc8da79530d4f7",
            "0de116d41931489b83c0105016ce7cec",
            "d87e3e92744a4a638d4d4ca8c03aaa45",
            "e971b5dc9f55400ca146ae4775d1b642"
          ]
        },
        "id": "cq9mi-Iq26Bj",
        "outputId": "301b182f-e54c-49ea-9d13-4c1358611c8c"
      },
      "source": [
        "from transformers import BertTokenizer\n",
        "tokenizer = BertTokenizer.from_pretrained(\n",
        "    \"beomi/kcbert-base\",\n",
        "    do_lower_case=False,\n",
        ")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b860e5773ed143cca217926091115ffb",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=249928.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "daa1ab91c2bb45f9b026a2a1fb9f49c3",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=49.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K0NCnzpRTdnJ"
      },
      "source": [
        "## 모델 초기화\n",
        "\n",
        "BERT(`kcbert-base`) 모델을 읽어들임"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FSVDbnFUTiXx",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 189,
          "referenced_widgets": [
            "dd7a14e1b71049ec8e30a213da049ddf",
            "d2a1150a2f4d4a6689702b5a67536333",
            "99bac06470884a5e9c8e192465fa28f5",
            "0c7b748dbafd40dab65fba80a1fdeb77",
            "9f0c4ef6f5de4ea1919b9a2e5d211f2c",
            "6a80ae9b74204ceb95e74cfc73b4c18f",
            "04d2881abf924ce0aa9a9516880472aa",
            "0989be37fa3149a397a76ba7a8bd48af",
            "d802ddf1d1de490fbdfee58af5966a49",
            "0b097dd8923d43ab951d986351aa40f9",
            "904409fa2e7c4463a63e7583ab8726cd",
            "3dd25ca47d6e4fed8846ff370e39693d",
            "0e95fa4fd79e4a06bd7d9e64b305ef01",
            "7e88c1adb28343468ec4c95634b76784",
            "64d8c3a1d09a429483bf1776d97af3ec",
            "4ce0205c763f42c1980740cc44a5cb46"
          ]
        },
        "outputId": "a4813d89-d4a5-4b51-985e-87aa3add5ce7"
      },
      "source": [
        "from transformers import BertConfig, BertModel\n",
        "pretrained_model_config = BertConfig.from_pretrained(\n",
        "    \"beomi/kcbert-base\"\n",
        ")\n",
        "model = BertModel.from_pretrained(\n",
        "    \"beomi/kcbert-base\",\n",
        "    config=pretrained_model_config,\n",
        ")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "dd7a14e1b71049ec8e30a213da049ddf",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=619.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d802ddf1d1de490fbdfee58af5966a49",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=438218004.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at beomi/kcbert-base were not used when initializing BertModel: ['cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight']\n",
            "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sWboOauTTyXp"
      },
      "source": [
        "`pretrained_model_config`의 내용 확인"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tup5LreLT4vE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "898d0726-01c1-41a4-8a8f-194ba0efbfb3"
      },
      "source": [
        "pretrained_model_config"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BertConfig {\n",
              "  \"_name_or_path\": \"beomi/kcbert-base\",\n",
              "  \"architectures\": [\n",
              "    \"BertForMaskedLM\"\n",
              "  ],\n",
              "  \"attention_probs_dropout_prob\": 0.1,\n",
              "  \"directionality\": \"bidi\",\n",
              "  \"gradient_checkpointing\": false,\n",
              "  \"hidden_act\": \"gelu\",\n",
              "  \"hidden_dropout_prob\": 0.1,\n",
              "  \"hidden_size\": 768,\n",
              "  \"initializer_range\": 0.02,\n",
              "  \"intermediate_size\": 3072,\n",
              "  \"layer_norm_eps\": 1e-12,\n",
              "  \"max_position_embeddings\": 300,\n",
              "  \"model_type\": \"bert\",\n",
              "  \"num_attention_heads\": 12,\n",
              "  \"num_hidden_layers\": 12,\n",
              "  \"pad_token_id\": 0,\n",
              "  \"pooler_fc_size\": 768,\n",
              "  \"pooler_num_attention_heads\": 12,\n",
              "  \"pooler_num_fc_layers\": 3,\n",
              "  \"pooler_size_per_head\": 128,\n",
              "  \"pooler_type\": \"first_token_transform\",\n",
              "  \"position_embedding_type\": \"absolute\",\n",
              "  \"transformers_version\": \"4.6.1\",\n",
              "  \"type_vocab_size\": 2,\n",
              "  \"use_cache\": true,\n",
              "  \"vocab_size\": 30000\n",
              "}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8Y2JYqRGUFll"
      },
      "source": [
        "## 모델 입력값 만들기\n",
        "\n",
        "문장 2개를 모델 입력값으로 지정"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9Rk2Ga65USFb"
      },
      "source": [
        "sentences = [\"안녕하세요\", \"안녕하십니까\", \"안녕하십니까.\"]\n",
        "features = tokenizer(\n",
        "    sentences,\n",
        "    max_length=10,\n",
        "    padding=\"max_length\",\n",
        "    truncation=True,\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wLq1JRVJUb7U"
      },
      "source": [
        "`features`의 내용을 확인"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XZl8OKdsVIQg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "66de0c7e-d3a8-4f9b-ebc3-2b49ba29609c"
      },
      "source": [
        "features.keys()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dict_keys(['input_ids', 'token_type_ids', 'attention_mask'])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LzkJ31vQVbjl",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1dd39af5-9c3d-4688-de6e-f26456848cbe"
      },
      "source": [
        "features['input_ids']"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[[2, 19017, 8482, 3, 0, 0, 0, 0, 0, 0],\n",
              " [2, 19017, 22796, 3, 0, 0, 0, 0, 0, 0],\n",
              " [2, 19017, 22796, 17, 3, 0, 0, 0, 0, 0]]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Lc_71OjafQgS",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "630eb839-6506-401f-b1e4-d3e99723cf20"
      },
      "source": [
        "features['attention_mask']"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[[1, 1, 1, 1, 0, 0, 0, 0, 0, 0],\n",
              " [1, 1, 1, 1, 0, 0, 0, 0, 0, 0],\n",
              " [1, 1, 1, 1, 1, 0, 0, 0, 0, 0]]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fdv1TGOHfQpm",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "eceff107-25f3-4400-b4b9-286802ddea69"
      },
      "source": [
        "features['token_type_ids']"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[[0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
              " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
              " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wR-3V6zIVsPp"
      },
      "source": [
        "## BERT 임베딩 추출\n",
        "\n",
        "위에서 만든 `features`를 파이토치 텐서(tensor)로 변환"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FdvUPuJoV3w3"
      },
      "source": [
        "import torch\n",
        "features = {k: torch.tensor(v) for k, v in features.items()}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W0S2EbdkWIdq"
      },
      "source": [
        "BERT 모델에 `features`를 입력하여 계산"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YOWkywHaWOL6"
      },
      "source": [
        "outputs = model(**features)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Fog_9z0SqHog"
      },
      "source": [
        "BERT 마지막 레이어의 단어 수준 출력 벡터 확인"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-wRQNlJMqMjh",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cf91d293-8c65-429f-a997-e7c461d5bcda"
      },
      "source": [
        "outputs.last_hidden_state"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[[-0.6969, -0.8248,  1.7512,  ..., -0.3732,  0.7399,  1.1907],\n",
              "         [-1.4803, -0.4398,  0.9444,  ..., -0.7405, -0.0211,  1.3064],\n",
              "         [-1.4299, -0.5033, -0.2069,  ...,  0.1285, -0.2611,  1.6057],\n",
              "         ...,\n",
              "         [-1.4406,  0.3431,  1.4043,  ..., -0.0565,  0.8450, -0.2170],\n",
              "         [-1.3625, -0.2404,  1.1757,  ...,  0.8876, -0.1054,  0.0734],\n",
              "         [-1.4244,  0.1518,  1.2920,  ...,  0.0245,  0.7572,  0.0080]],\n",
              "\n",
              "        [[-0.4433, -0.9240,  1.7883,  ..., -0.8659,  0.9727,  0.9249],\n",
              "         [-1.6224, -0.0682,  0.1769,  ..., -0.8680,  0.7786,  1.9937],\n",
              "         [-1.1414, -0.1960, -0.7010,  ...,  0.6878,  0.3851,  0.4984],\n",
              "         ...,\n",
              "         [-1.0095,  0.0397,  1.5963,  ...,  0.8457,  0.3166, -0.0534],\n",
              "         [-1.2240,  0.4880,  1.3206,  ...,  1.4305,  0.7170,  0.7261],\n",
              "         [-1.2351, -0.0513,  1.4715,  ...,  0.6764,  0.1578, -0.1697]],\n",
              "\n",
              "        [[-0.1236, -1.0882,  1.5019,  ..., -1.0845,  0.5248,  0.6436],\n",
              "         [-1.4286, -0.1186,  0.1144,  ..., -0.7928,  0.7173,  1.9609],\n",
              "         [-0.8282, -0.1456, -0.6139,  ...,  0.9649,  0.8140,  0.4001],\n",
              "         ...,\n",
              "         [-0.1945, -0.2689,  1.6015,  ...,  1.1928, -0.0607, -0.3073],\n",
              "         [-0.3753, -0.2505,  1.3539,  ...,  1.4079,  0.8315,  0.2978],\n",
              "         [-0.9675, -0.1184,  1.1998,  ...,  1.4431, -0.2637, -0.1747]]],\n",
              "       grad_fn=<NativeLayerNormBackward>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nruSHwDE6kj1",
        "outputId": "5942b442-9b90-4b3c-9e52-085cfab6780d"
      },
      "source": [
        "print(outputs.last_hidden_state.shape)\n",
        "print(outputs.last_hidden_state[0, 1, :30])\n",
        "print(outputs.last_hidden_state[1, 1, :30])\n",
        "print(outputs.last_hidden_state[2, 1, :30])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([3, 10, 768])\n",
            "tensor([-1.4803, -0.4398,  0.9444,  0.4441,  0.6319,  0.8464,  0.5650,  0.2841,\n",
            "         0.3534,  2.0140, -1.3429, -0.4704, -0.6285, -0.6812,  0.5594,  0.0941,\n",
            "         0.6915,  0.0828,  0.4902, -0.3403,  0.3471,  0.7107, -0.4473, -0.3360,\n",
            "         0.8616,  0.0462,  0.2277,  0.8386,  0.3912, -0.4810],\n",
            "       grad_fn=<SliceBackward>)\n",
            "tensor([-1.6224, -0.0682,  0.1769,  0.4605,  0.1461,  0.4836,  1.0702,  0.3585,\n",
            "         0.5402,  1.9740, -1.0956, -0.7817, -0.8804, -0.5016, -0.1314,  0.3771,\n",
            "         1.0445, -0.4988, -0.0756, -0.0102,  0.9213,  0.3548, -0.7024,  1.2910,\n",
            "         1.3515, -0.2282,  0.4545,  0.5006, -0.3627, -0.8802],\n",
            "       grad_fn=<SliceBackward>)\n",
            "tensor([-1.4286, -0.1186,  0.1144,  0.5986,  0.2569,  0.4758,  0.8051,  0.3198,\n",
            "         0.7381,  2.0553, -1.1412, -0.7723, -1.1986, -0.5407, -0.1609,  0.3903,\n",
            "         0.9213, -0.5395, -0.0479,  0.1581,  0.9003,  0.3982, -0.5360,  1.4631,\n",
            "         1.3810, -0.3041,  0.6067,  0.4754, -0.0529, -0.6580],\n",
            "       grad_fn=<SliceBackward>)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AwDi0WNdrG7r"
      },
      "source": [
        "BERT 마지막 레이어의 CLS 벡터를 확인"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0FQ1nayZrLQl",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c25c4e68-46ba-4dde-ca87-5426550a361e"
      },
      "source": [
        "outputs.pooler_output"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[-0.1594,  0.0547,  0.1101,  ...,  0.2684,  0.1596, -0.9828],\n",
              "        [ 0.1988,  0.0735,  0.0183,  ...,  0.2965, -0.1216, -0.9984],\n",
              "        [ 0.1642,  0.0842,  0.0423,  ...,  0.3627,  0.0049, -0.9981]],\n",
              "       grad_fn=<TanhBackward>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z1GM97fo3nkz"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}