mozturk_BERT+CNN.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "mozturk_BERT+CNN.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "b79f73a68a3549179b0a28d41acb067d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_80e4b9309e674ce39595d28640944683",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_197f6716bea44b7f8712a27c5217690f",
              "IPY_MODEL_c2cb342c08b044d4934f5d04d4a03fb0",
              "IPY_MODEL_a3c0ec2c265a4b1189719079fde84ee5"
            ]
          }
        },
        "80e4b9309e674ce39595d28640944683": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "197f6716bea44b7f8712a27c5217690f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_090f80ebefed467a998d83f0eada2de4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_518431db07be4d2f85ece877c0ab3b1a"
          }
        },
        "c2cb342c08b044d4934f5d04d4a03fb0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1ebf8461fa474203b1861db163e162b8",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1233088,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1233088,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_84340cee678740f7a2bbab4514cdab48"
          }
        },
        "a3c0ec2c265a4b1189719079fde84ee5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_25358475d21644f58c114c3948c4d4e4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.18M/1.18M [00:00&lt;00:00, 2.89MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6b08b9749ea94a168527bec36aab4ee9"
          }
        },
        "090f80ebefed467a998d83f0eada2de4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "518431db07be4d2f85ece877c0ab3b1a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1ebf8461fa474203b1861db163e162b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "84340cee678740f7a2bbab4514cdab48": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "25358475d21644f58c114c3948c4d4e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6b08b9749ea94a168527bec36aab4ee9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9f4b604934744ac594d1cd5e5c157078": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b6d8c01bd6b043808463afe819fe9198",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5cae0cc9028e40d0b448c226e4f417ef",
              "IPY_MODEL_e9c82b6a72e44d33b49b816686bbb66e",
              "IPY_MODEL_5c1edd3718bd4cc0862ee12da1c0455b"
            ]
          }
        },
        "b6d8c01bd6b043808463afe819fe9198": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5cae0cc9028e40d0b448c226e4f417ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1b08ac7081ec475da37465086a5ae563",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e004865d6b8c44eda4a78eee9c849028"
          }
        },
        "e9c82b6a72e44d33b49b816686bbb66e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f16a92fa8d5441ebad578ac563eb7423",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 59,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 59,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7f8944b4d68c453293982f1e045d8442"
          }
        },
        "5c1edd3718bd4cc0862ee12da1c0455b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5e7fd79750ef4bb597434ff11b77a78a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 59.0/59.0 [00:00&lt;00:00, 1.75kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2617e83b2cb44e98a9f7da2789f99edb"
          }
        },
        "1b08ac7081ec475da37465086a5ae563": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e004865d6b8c44eda4a78eee9c849028": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f16a92fa8d5441ebad578ac563eb7423": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7f8944b4d68c453293982f1e045d8442": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5e7fd79750ef4bb597434ff11b77a78a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2617e83b2cb44e98a9f7da2789f99edb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f7eee83f75184c789be652d5b765a670": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a795de5f52a24ca895c7187da8f778c8",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8424539724e94f468b69630db3a1f778",
              "IPY_MODEL_ac31619dbf604207a309c842a1096ab3",
              "IPY_MODEL_da2e0b84d36344bb85b1dacd1eca99a4"
            ]
          }
        },
        "a795de5f52a24ca895c7187da8f778c8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8424539724e94f468b69630db3a1f778": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9d2caa37b05a47c4bfd33d2bc61d2eed",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bd3b728130154cdbb3ac52eefc3f259d"
          }
        },
        "ac31619dbf604207a309c842a1096ab3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_14747e1a0e044ca48d624f8171f51cb6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 386,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 386,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bc0ba4829b95489d95c27c0645d456ea"
          }
        },
        "da2e0b84d36344bb85b1dacd1eca99a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5d349d87736a452d82d159fc2aad8e8c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 386/386 [00:00&lt;00:00, 11.7kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8bf9bd3fe4e94c88a00f5c526fca9235"
          }
        },
        "9d2caa37b05a47c4bfd33d2bc61d2eed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bd3b728130154cdbb3ac52eefc3f259d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "14747e1a0e044ca48d624f8171f51cb6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bc0ba4829b95489d95c27c0645d456ea": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5d349d87736a452d82d159fc2aad8e8c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8bf9bd3fe4e94c88a00f5c526fca9235": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zXGbkXbHdWvV"
      },
      "outputs": [],
      "source": [
        "import json\n",
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "import openpyxl\n",
        "from pathlib import Path\n",
        "import numpy as np\n",
        "from gensim.models import KeyedVectors\n",
        "\n",
        "df = pd.read_excel(\"/content/drive/MyDrive/Colab Notebooks/models/last_data.xlsx\") "
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df = df[[\"tweet\",\"label\"]]\n",
        "sentences = df[\"tweet\"].values\n",
        "y = df[\"label\"].values\n",
        "\n",
        "training_texts, test_texts, training_labels, test_labels = train_test_split(sentences, y, test_size=0.25, random_state=1000)"
      ],
      "metadata": {
        "id": "hIxjzxXzd9yj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "train, test = train_test_split(df, test_size=0.2)\n",
        "train,val = train_test_split(train, test_size=0.2)"
      ],
      "metadata": {
        "id": "nSWatUTTqiIE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers\n",
        "import transformers\n",
        "from transformers import BertTokenizer\n",
        "from torch.utils.data import TensorDataset, random_split\n",
        "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
        "from transformers import BertForSequenceClassification, AdamW, BertConfig\n",
        "from transformers import get_linear_schedule_with_warmup\n",
        "import torch\n",
        "from transformers import BertModel"
      ],
      "metadata": {
        "id": "TFQc7zUPeBJL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Download the pre-trained Turkish BERT Model"
      ],
      "metadata": {
        "id": "uIglBAF7e3Hh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = BertTokenizer.from_pretrained('dbmdz/bert-base-turkish-128k-uncased', do_lower_case=True)\n",
        "sentences = df.tweet.values\n",
        "max_len = 50"
      ],
      "metadata": {
        "id": "290ipCUHeNnd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 113,
          "referenced_widgets": [
            "b79f73a68a3549179b0a28d41acb067d",
            "80e4b9309e674ce39595d28640944683",
            "197f6716bea44b7f8712a27c5217690f",
            "c2cb342c08b044d4934f5d04d4a03fb0",
            "a3c0ec2c265a4b1189719079fde84ee5",
            "090f80ebefed467a998d83f0eada2de4",
            "518431db07be4d2f85ece877c0ab3b1a",
            "1ebf8461fa474203b1861db163e162b8",
            "84340cee678740f7a2bbab4514cdab48",
            "25358475d21644f58c114c3948c4d4e4",
            "6b08b9749ea94a168527bec36aab4ee9",
            "9f4b604934744ac594d1cd5e5c157078",
            "b6d8c01bd6b043808463afe819fe9198",
            "5cae0cc9028e40d0b448c226e4f417ef",
            "e9c82b6a72e44d33b49b816686bbb66e",
            "5c1edd3718bd4cc0862ee12da1c0455b",
            "1b08ac7081ec475da37465086a5ae563",
            "e004865d6b8c44eda4a78eee9c849028",
            "f16a92fa8d5441ebad578ac563eb7423",
            "7f8944b4d68c453293982f1e045d8442",
            "5e7fd79750ef4bb597434ff11b77a78a",
            "2617e83b2cb44e98a9f7da2789f99edb",
            "f7eee83f75184c789be652d5b765a670",
            "a795de5f52a24ca895c7187da8f778c8",
            "8424539724e94f468b69630db3a1f778",
            "ac31619dbf604207a309c842a1096ab3",
            "da2e0b84d36344bb85b1dacd1eca99a4",
            "9d2caa37b05a47c4bfd33d2bc61d2eed",
            "bd3b728130154cdbb3ac52eefc3f259d",
            "14747e1a0e044ca48d624f8171f51cb6",
            "bc0ba4829b95489d95c27c0645d456ea",
            "5d349d87736a452d82d159fc2aad8e8c",
            "8bf9bd3fe4e94c88a00f5c526fca9235"
          ]
        },
        "outputId": "6acb6bd5-c012-4191-9ab0-9ef7217e3520"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b79f73a68a3549179b0a28d41acb067d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.18M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9f4b604934744ac594d1cd5e5c157078",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/59.0 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f7eee83f75184c789be652d5b765a670",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/386 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model = BertModel.from_pretrained('dbmdz/bert-base-turkish-128k-uncased',output_hidden_states=True)\n",
        "model.eval()"
      ],
      "metadata": {
        "id": "XDJOtGGQi_M1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Last 4 Layer of BERT + CNN"
      ],
      "metadata": {
        "id": "BU69nJxKfCho"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def last_4_emb(text):\n",
        "  tokenized_text = tokenizer.tokenize(text)\n",
        "  # Map the token strings to their vocabulary indeces.\n",
        "  indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)\n",
        "  # Display the words with their indeces.\n",
        "  \n",
        "  tokens_tensor = torch.tensor([indexed_tokens])\n",
        "  \n",
        "  with torch.no_grad():\n",
        "    outputs = model(tokens_tensor)\n",
        "    last_hidden_state = outputs[0]\n",
        "    word_embed_1 = last_hidden_state\n",
        "    hidden_states = outputs[2]\n",
        "    # initial embeddings can be taken from 0th layer of hidden states\n",
        "    word_embed_2 = hidden_states[0]\n",
        "    # sum of all hidden states\n",
        "    word_embed_3 = torch.stack(hidden_states).sum(0)\n",
        "    # sum of second to last layer\n",
        "    word_embed_4 = torch.stack(hidden_states[2:]).sum(0)\n",
        "    # sum of last four layer\n",
        "    word_embed_5 = torch.stack(hidden_states[-4:]).sum(0)\n",
        "    # concatenate last four layers\n",
        "    word_embed_6 = torch.cat([hidden_states[i] for i in [-1,-2,-3,-4]], dim=-1)\n",
        "\n",
        "    sentence_emb = np.array(word_embed_6[0].sum(0))\n",
        "\n",
        "    return sentence_emb\n",
        "  \n",
        "\n",
        "  \n"
      ],
      "metadata": {
        "id": "xN39S4i0nhPN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_emb = []\n",
        "\n",
        "for tweet in train.tweet:\n",
        "  train_emb.append(last_4_emb(tweet))\n",
        "\n",
        "val_emb = []\n",
        "\n",
        "for tweet in val.tweet:\n",
        "  val_emb.append(last_4_emb(tweet))\n",
        "\n",
        "test_emb = []\n",
        "\n",
        "for tweet in test.tweet:\n",
        "  test_emb.append(last_4_emb(tweet))\n",
        "\n",
        "y_train = train.label\n",
        "y_test = test.label\n",
        "y_val = val.label\n",
        "\n",
        "x_train = np.array(train_emb)\n",
        "x_val = np.array(val_emb)\n",
        "\n",
        "X_train = x_train.reshape(-1, 3072, 1)\n",
        "X_val = x_val.reshape(-1, 3072, 1)"
      ],
      "metadata": {
        "id": "hjNuVmFyoBnM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Training CNN with the BERT Embeddings"
      ],
      "metadata": {
        "id": "JmKeSLtJfUFN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import keras\n",
        "from keras import models\n",
        "from keras.models import Sequential\n",
        "from keras import layers\n",
        "from keras.layers import Embedding, Bidirectional, Dense, LSTM, Conv1D, MaxPooling1D, Flatten, Reshape, TimeDistributed\n",
        "from keras import optimizers\n",
        "\n",
        "model1 = Sequential()\n",
        "\n",
        "model1.add(Conv1D(256, 3, activation='relu')) # input_shape = (768*3,1)\n",
        "\n",
        "model1.add(Conv1D(128, 3,  activation='relu'))\n",
        "\n",
        "model1.add(Conv1D(128, 3,  activation='relu'))\n",
        "\n",
        "\n",
        "model1.add(MaxPooling1D(5))\n",
        "\n",
        "# flat\n",
        "model1.add(Flatten())\n",
        "\n",
        "model1.add(Dense(3, activation='softmax'))\n",
        "\n",
        "model1.compile(loss='sparse_categorical_crossentropy',\n",
        "              optimizer=\"adam\",\n",
        "              metrics=['accuracy'])\n",
        "\n",
        "history  = model1.fit(np.array(X_train), np.array(y_train),\n",
        " epochs= 30,\n",
        " batch_size = 200,\n",
        " validation_data = (np.array(X_val), np.array(y_val))\n",
        ")\n"
      ],
      "metadata": {
        "id": "87uOLZmarTnE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "path = '/content/drive/MyDrive/Colab Notebooks/models/BERT_last4_CNN.pt'\n",
        "torch.save(model1, path)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WmOkewjE1tMf",
        "outputId": "42ef1e56-b890-40c2-ffe0-9184a77030a7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:Assets written to: ram://ced7fb2a-694a-4724-80dc-fbec6eae01de/assets\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "X_test = np.array(test_emb)\n",
        "X_test = X_test.reshape((-1, 768*4, 1))\n",
        "predict_x=model1.predict(X_test) \n",
        "prediction=np.argmax(predict_x,axis=1)"
      ],
      "metadata": {
        "id": "sJ7lvegSvu8N"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn import metrics\n",
        "\n",
        "print(metrics.confusion_matrix(y_test, prediction))\n",
        "\n",
        "# Print the precision and recall, among other metrics\n",
        "print(metrics.classification_report(y_test, prediction, digits=3))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vg4lVjvVv-YD",
        "outputId": "cf7c9151-edb9-41c7-f5fc-9de78cf8faad"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[ 74  71   8]\n",
            " [ 38 187  34]\n",
            " [ 10  33  68]]\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0      0.607     0.484     0.538       153\n",
            "           1      0.643     0.722     0.680       259\n",
            "           2      0.618     0.613     0.615       111\n",
            "\n",
            "    accuracy                          0.629       523\n",
            "   macro avg      0.622     0.606     0.611       523\n",
            "weighted avg      0.627     0.629     0.625       523\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Last Layer of BERT + CNN"
      ],
      "metadata": {
        "id": "79CCaE4mfhCj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def last_emb(text):\n",
        "  tokenized_text = tokenizer.tokenize(text)\n",
        "  # Map the token strings to their vocabulary indeces.\n",
        "  indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)\n",
        "  # Display the words with their indeces.\n",
        "  \n",
        "  tokens_tensor = torch.tensor([indexed_tokens])\n",
        "  \n",
        "  with torch.no_grad():\n",
        "    outputs = model(tokens_tensor)\n",
        "    last_hidden_state = outputs[0]\n",
        "    sentence_emb = np.array(last_hidden_state[0].sum(0))\n",
        "\n",
        "    return sentence_emb"
      ],
      "metadata": {
        "id": "3TGZeJBy2EH2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_emb = []\n",
        "\n",
        "for tweet in train.tweet:\n",
        "  train_emb.append(last_emb(tweet))"
      ],
      "metadata": {
        "id": "QbQujfHo2IJW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "val_emb = []\n",
        "\n",
        "for tweet in val.tweet:\n",
        "  val_emb.append(last_emb(tweet))"
      ],
      "metadata": {
        "id": "VQheLOPr4d1W"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "test_emb = []\n",
        "\n",
        "for tweet in test.tweet:\n",
        "  test_emb.append(last_emb(tweet))"
      ],
      "metadata": {
        "id": "LZOHrBJ34eYa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "y_train = train.label\n",
        "y_test = test.label\n",
        "y_val = val.label\n",
        "\n",
        "x_train = np.array(train_emb)\n",
        "x_val = np.array(val_emb)\n",
        "\n",
        "X_train = x_train.reshape(-1, 768, 1)\n",
        "X_val = x_val.reshape(-1, 768, 1)"
      ],
      "metadata": {
        "id": "61NtrAwG5VnB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Training the CNN with the embeddings from BERT"
      ],
      "metadata": {
        "id": "bRNe-UExfm13"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import keras\n",
        "from keras import models\n",
        "from keras.models import Sequential\n",
        "from keras import layers\n",
        "from keras.layers import Embedding, Bidirectional, Dense, LSTM, Conv1D, MaxPooling1D, Flatten, Reshape, TimeDistributed\n",
        "from keras import optimizers\n",
        "\n",
        "model2 = Sequential()\n",
        "\n",
        "model2.add(Conv1D(256, 3, activation='relu')) # input_shape = (768,1)\n",
        "\n",
        "model2.add(Conv1D(128, 3,  activation='relu'))\n",
        "\n",
        "model2.add(Conv1D(128, 3,  activation='relu'))\n",
        "\n",
        "\n",
        "\n",
        "model2.add(MaxPooling1D(5))\n",
        "\n",
        "# flat\n",
        "model2.add(Flatten())\n",
        "\n",
        "model2.add(Dense(3, activation='softmax'))\n",
        "\n",
        "model2.compile(loss='sparse_categorical_crossentropy',\n",
        "              optimizer=\"adam\",\n",
        "              metrics=['accuracy'])\n",
        "\n",
        "history  = model2.fit(np.array(X_train), np.array(y_train),\n",
        " epochs= 30,\n",
        " batch_size = 200,\n",
        " validation_data = (np.array(X_val), np.array(y_val))\n",
        ")"
      ],
      "metadata": {
        "id": "j5cCBRWP5osZ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "281c1fd3-beb4-486c-ceb6-000028f73136"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/30\n",
            "9/9 [==============================] - 2s 124ms/step - loss: 2.1179 - accuracy: 0.3980 - val_loss: 1.0575 - val_accuracy: 0.4737\n",
            "Epoch 2/30\n",
            "9/9 [==============================] - 1s 71ms/step - loss: 1.0464 - accuracy: 0.4333 - val_loss: 1.0428 - val_accuracy: 0.4737\n",
            "Epoch 3/30\n",
            "9/9 [==============================] - 1s 71ms/step - loss: 1.0142 - accuracy: 0.5057 - val_loss: 1.0395 - val_accuracy: 0.4737\n",
            "Epoch 4/30\n",
            "9/9 [==============================] - 1s 72ms/step - loss: 1.0034 - accuracy: 0.4985 - val_loss: 1.0754 - val_accuracy: 0.4737\n",
            "Epoch 5/30\n",
            "9/9 [==============================] - 1s 70ms/step - loss: 0.9610 - accuracy: 0.5554 - val_loss: 0.9445 - val_accuracy: 0.5574\n",
            "Epoch 6/30\n",
            "9/9 [==============================] - 1s 70ms/step - loss: 0.8694 - accuracy: 0.5937 - val_loss: 0.8921 - val_accuracy: 0.5766\n",
            "Epoch 7/30\n",
            "9/9 [==============================] - 1s 71ms/step - loss: 0.7954 - accuracy: 0.6338 - val_loss: 0.8491 - val_accuracy: 0.5957\n",
            "Epoch 8/30\n",
            "9/9 [==============================] - 1s 71ms/step - loss: 0.7549 - accuracy: 0.6511 - val_loss: 0.8373 - val_accuracy: 0.6268\n",
            "Epoch 9/30\n",
            "9/9 [==============================] - 1s 71ms/step - loss: 0.7195 - accuracy: 0.6774 - val_loss: 0.8468 - val_accuracy: 0.6053\n",
            "Epoch 10/30\n",
            "9/9 [==============================] - 1s 71ms/step - loss: 0.6505 - accuracy: 0.7325 - val_loss: 0.8646 - val_accuracy: 0.6244\n",
            "Epoch 11/30\n",
            "9/9 [==============================] - 1s 71ms/step - loss: 0.6213 - accuracy: 0.7271 - val_loss: 0.9248 - val_accuracy: 0.6292\n",
            "Epoch 12/30\n",
            "9/9 [==============================] - 1s 71ms/step - loss: 0.5804 - accuracy: 0.7493 - val_loss: 0.8726 - val_accuracy: 0.6507\n",
            "Epoch 13/30\n",
            "9/9 [==============================] - 1s 71ms/step - loss: 0.5059 - accuracy: 0.8001 - val_loss: 0.9147 - val_accuracy: 0.6435\n",
            "Epoch 14/30\n",
            "9/9 [==============================] - 1s 72ms/step - loss: 0.4764 - accuracy: 0.8055 - val_loss: 0.9751 - val_accuracy: 0.6459\n",
            "Epoch 15/30\n",
            "9/9 [==============================] - 1s 71ms/step - loss: 0.4431 - accuracy: 0.8259 - val_loss: 1.0337 - val_accuracy: 0.6388\n",
            "Epoch 16/30\n",
            "9/9 [==============================] - 1s 73ms/step - loss: 0.3983 - accuracy: 0.8510 - val_loss: 1.0962 - val_accuracy: 0.6100\n",
            "Epoch 17/30\n",
            "9/9 [==============================] - 1s 71ms/step - loss: 0.4098 - accuracy: 0.8468 - val_loss: 1.0083 - val_accuracy: 0.6483\n",
            "Epoch 18/30\n",
            "9/9 [==============================] - 1s 72ms/step - loss: 0.3216 - accuracy: 0.8863 - val_loss: 1.1813 - val_accuracy: 0.6364\n",
            "Epoch 19/30\n",
            "9/9 [==============================] - 1s 71ms/step - loss: 0.2985 - accuracy: 0.8899 - val_loss: 1.2238 - val_accuracy: 0.6244\n",
            "Epoch 20/30\n",
            "9/9 [==============================] - 1s 73ms/step - loss: 0.2564 - accuracy: 0.9090 - val_loss: 1.2206 - val_accuracy: 0.6316\n",
            "Epoch 21/30\n",
            "9/9 [==============================] - 1s 72ms/step - loss: 0.2358 - accuracy: 0.9258 - val_loss: 1.2359 - val_accuracy: 0.5981\n",
            "Epoch 22/30\n",
            "9/9 [==============================] - 1s 71ms/step - loss: 0.2037 - accuracy: 0.9336 - val_loss: 1.3186 - val_accuracy: 0.6053\n",
            "Epoch 23/30\n",
            "9/9 [==============================] - 1s 72ms/step - loss: 0.1562 - accuracy: 0.9509 - val_loss: 1.4405 - val_accuracy: 0.6364\n",
            "Epoch 24/30\n",
            "9/9 [==============================] - 1s 73ms/step - loss: 0.1336 - accuracy: 0.9665 - val_loss: 1.5783 - val_accuracy: 0.6340\n",
            "Epoch 25/30\n",
            "9/9 [==============================] - 1s 72ms/step - loss: 0.1184 - accuracy: 0.9659 - val_loss: 1.5984 - val_accuracy: 0.6100\n",
            "Epoch 26/30\n",
            "9/9 [==============================] - 1s 72ms/step - loss: 0.0896 - accuracy: 0.9785 - val_loss: 1.7033 - val_accuracy: 0.6100\n",
            "Epoch 27/30\n",
            "9/9 [==============================] - 1s 72ms/step - loss: 0.0874 - accuracy: 0.9791 - val_loss: 1.8278 - val_accuracy: 0.6483\n",
            "Epoch 28/30\n",
            "9/9 [==============================] - 1s 72ms/step - loss: 0.0629 - accuracy: 0.9880 - val_loss: 1.8914 - val_accuracy: 0.6124\n",
            "Epoch 29/30\n",
            "9/9 [==============================] - 1s 72ms/step - loss: 0.0540 - accuracy: 0.9910 - val_loss: 1.9760 - val_accuracy: 0.6292\n",
            "Epoch 30/30\n",
            "9/9 [==============================] - 1s 72ms/step - loss: 0.0416 - accuracy: 0.9946 - val_loss: 2.0655 - val_accuracy: 0.6077\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "X_test = np.array(test_emb)\n",
        "X_test = X_test.reshape((-1, 768, 1))\n",
        "predict_x=model2.predict(X_test) \n",
        "prediction=np.argmax(predict_x,axis=1)"
      ],
      "metadata": {
        "id": "305Q6Xdf7Y83"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn import metrics\n",
        "\n",
        "print(metrics.confusion_matrix(y_test, prediction))\n",
        "\n",
        "# Print the precision and recall, among other metrics\n",
        "print(metrics.classification_report(y_test, prediction, digits=3))"
      ],
      "metadata": {
        "id": "nbsaxWs97c3R",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a079361e-795a-4399-efb9-3efff96577cc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[ 77  68   8]\n",
            " [ 57 176  26]\n",
            " [ 17  28  66]]\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0      0.510     0.503     0.507       153\n",
            "           1      0.647     0.680     0.663       259\n",
            "           2      0.660     0.595     0.626       111\n",
            "\n",
            "    accuracy                          0.610       523\n",
            "   macro avg      0.606     0.592     0.598       523\n",
            "weighted avg      0.610     0.610     0.609       523\n",
            "\n"
          ]
        }
      ]
    }
  ]
}