Task 3.2 a.) -  Binary classifier over the dataset .ipynb

{"cells":[{"cell_type":"markdown","metadata":{"id":"JRkv266S972W"},"source":["#Task 3.2 a.) -  Binary classifier over the dataset\n","By Abdulwahab Adeshina Abdulliameed\n"]},{"cell_type":"markdown","metadata":{"id":"82nnS1p_J8vA"},"source":["**Setting Up PySpark in Colab**"]},{"cell_type":"code","execution_count":1,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":57007,"status":"ok","timestamp":1648912730024,"user":{"displayName":"Abdulwahab Abdulliameed","userId":"10732377996348789858"},"user_tz":-60},"id":"COCOvti5KySL","outputId":"042e9d92-3bc2-48e5-f566-9785c2ac678b"},"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting pyspark\n","  Downloading pyspark-3.2.1.tar.gz (281.4 MB)\n","\u001b[K     |████████████████████████████████| 281.4 MB 34 kB/s \n","\u001b[?25hCollecting py4j==0.10.9.3\n","  Downloading py4j-0.10.9.3-py2.py3-none-any.whl (198 kB)\n","\u001b[K     |████████████████████████████████| 198 kB 51.4 MB/s \n","\u001b[?25hBuilding wheels for collected packages: pyspark\n","  Building wheel for pyspark (setup.py) ... \u001b[?25l\u001b[?25hdone\n","  Created wheel for pyspark: filename=pyspark-3.2.1-py2.py3-none-any.whl size=281853642 sha256=a48f379677725b48e2c617df8fc279ba975474a79d1296631a49cf5415e526d3\n","  Stored in directory: /root/.cache/pip/wheels/9f/f5/07/7cd8017084dce4e93e84e92efd1e1d5334db05f2e83bcef74f\n","Successfully built pyspark\n","Installing collected packages: py4j, pyspark\n","Successfully installed py4j-0.10.9.3 pyspark-3.2.1\n"]}],"source":["#Install pyspark\n","!pip install -q findspark\n","!pip install pyspark"]},{"cell_type":"code","execution_count":2,"metadata":{"id":"TqHTb6K-Dc29","executionInfo":{"status":"ok","timestamp":1648912743210,"user_tz":-60,"elapsed":9985,"user":{"displayName":"Abdulwahab Abdulliameed","userId":"10732377996348789858"}}},"outputs":[],"source":["#Running a local SparkSession\n","import findspark\n","findspark.init()\n","from pyspark.sql import SparkSession\n","spark = SparkSession.builder\\\n",".master(\"local\")\\\n",".appName(\"Colab\")\\\n",".config('spark.ui.port', '4050')\\\n",".getOrCreate()"]},{"cell_type":"code","execution_count":3,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":339,"status":"ok","timestamp":1648912783871,"user":{"displayName":"Abdulwahab Abdulliameed","userId":"10732377996348789858"},"user_tz":-60},"id":"V6UKIfaBwy1R","outputId":"8a6210f3-240f-4665-d4bd-18942b88ba64"},"outputs":[{"output_type":"stream","name":"stdout","text":["3.2.1\n"]}],"source":["# Check the pyspark version\n","import pyspark\n","print(pyspark.__version__)"]},{"cell_type":"code","execution_count":4,"metadata":{"id":"7wVS5hGWUmdN","executionInfo":{"status":"ok","timestamp":1648912788373,"user_tz":-60,"elapsed":361,"user":{"displayName":"Abdulwahab Abdulliameed","userId":"10732377996348789858"}}},"outputs":[],"source":["# Numerical analysis package\n","import numpy as np\n","# Package to work with data\n","import pandas as pd\n","# Package for Matplotlib Pyplot\n","# Pyplot is a collection of functions in the popular visualization package Matplotlib.\n","import matplotlib as mpl\n","import matplotlib.pyplot as plt\n","#PySpark SQL Package \n","import pyspark.sql.functions as F\n","import pyspark.sql\n","from pyspark.context import SparkContext\n","from pyspark.sql.functions import * "]},{"cell_type":"markdown","metadata":{"id":"OVr_jzwa_j_q"},"source":["**Connect/Mount Google Drive to Colab**"]},{"cell_type":"code","execution_count":5,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":22679,"status":"ok","timestamp":1648912814464,"user":{"displayName":"Abdulwahab Abdulliameed","userId":"10732377996348789858"},"user_tz":-60},"id":"UqwaPjGCQWCS","outputId":"e1597370-8068-41df-a4e5-642140059673"},"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/drive\n"]}],"source":["#Connect/Mount Google Drive to Colab\n","from google.colab import drive\n","drive.mount('/content/drive')"]},{"cell_type":"markdown","metadata":{"id":"dc46wz78ADHt"},"source":["**Define the UNSW-NB15 Dataframe**"]},{"cell_type":"code","execution_count":6,"metadata":{"id":"koboL_Ic-Wya","executionInfo":{"status":"ok","timestamp":1648912903882,"user_tz":-60,"elapsed":37898,"user":{"displayName":"Abdulwahab Abdulliameed","userId":"10732377996348789858"}}},"outputs":[],"source":["#Define the UNSW-NB15 Dataframe\n","UNSWNB15 = spark.read.csv('/content/drive/MyDrive/UEL-UNICAF/Module 3: Big Data Analytics/Assessment/UNSW-NB15.csv', inferSchema=True )"]},{"cell_type":"code","execution_count":7,"metadata":{"id":"iBI8fkBGLpLk","executionInfo":{"status":"ok","timestamp":1648912998287,"user_tz":-60,"elapsed":2015,"user":{"displayName":"Abdulwahab Abdulliameed","userId":"10732377996348789858"}}},"outputs":[],"source":["#Rename the Columns Header\n","UNSWNB15 = UNSWNB15.withColumnRenamed('_c0','srcip') \\\n",".withColumnRenamed('_c1','sport') \\\n",".withColumnRenamed('_c2','dstip') \\\n",".withColumnRenamed('_c3','dsport') \\\n",".withColumnRenamed('_c4','proto') \\\n",".withColumnRenamed('_c5','state') \\\n",".withColumnRenamed('_c6','dur') \\\n",".withColumnRenamed('_c7','sbytes') \\\n",".withColumnRenamed('_c8','dbytes') \\\n",".withColumnRenamed('_c9','sttl') \\\n",".withColumnRenamed('_c10','dttl') \\\n",".withColumnRenamed('_c11','sloss') \\\n",".withColumnRenamed('_c12','dloss') \\\n",".withColumnRenamed('_c13','service') \\\n",".withColumnRenamed('_c14','Sload') \\\n",".withColumnRenamed('_c15','Dload') \\\n",".withColumnRenamed('_c16','Spkts') \\\n",".withColumnRenamed('_c17','Dpkts') \\\n",".withColumnRenamed('_c18','swin') \\\n",".withColumnRenamed('_c19','dwin') \\\n",".withColumnRenamed('_c20','stcpb') \\\n",".withColumnRenamed('_c21','dtcpb') \\\n",".withColumnRenamed('_c22','smeansz') \\\n",".withColumnRenamed('_c23','dmeansz') \\\n",".withColumnRenamed('_c24','trans_depth') \\\n",".withColumnRenamed('_c25','res_bdy_len') \\\n",".withColumnRenamed('_c26','Sjit') \\\n",".withColumnRenamed('_c27','Djit') \\\n",".withColumnRenamed('_c28','Stime') \\\n",".withColumnRenamed('_c29','Ltime') \\\n",".withColumnRenamed('_c30','Spkt') \\\n",".withColumnRenamed('_c31','Dpkt') \\\n",".withColumnRenamed('_c32','tcprtt') \\\n",".withColumnRenamed('_c33','synack') \\\n",".withColumnRenamed('_c34','ackdat') \\\n",".withColumnRenamed('_c35','is_sm_ips_ports') \\\n",".withColumnRenamed('_c36','ct_state_ttl') \\\n",".withColumnRenamed('_c37','ct_flw_http_mthd') \\\n",".withColumnRenamed('_c38','is_ftp_login') \\\n",".withColumnRenamed('_c39','ct_ftp_cmd') \\\n",".withColumnRenamed('_c40','ct_srv_src') \\\n",".withColumnRenamed('_c41','ct_srv_dst') \\\n",".withColumnRenamed('_c42','ct_dst_ltm') \\\n",".withColumnRenamed('_c43','ct_src_ ltm') \\\n",".withColumnRenamed('_c44','ct_src_dport_ltm') \\\n",".withColumnRenamed('_c45','ct_dst_sport_ltm') \\\n",".withColumnRenamed('_c46','ct_dst_src_ltm') \\\n",".withColumnRenamed('_c47','attack_cat') \\\n",".withColumnRenamed('_c48','Label') \\"]},{"cell_type":"markdown","metadata":{"id":"PRHrtpRhAM1U"},"source":["**Working with PySpark**"]},{"cell_type":"code","execution_count":8,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":388,"status":"ok","timestamp":1648913011861,"user":{"displayName":"Abdulwahab Abdulliameed","userId":"10732377996348789858"},"user_tz":-60},"id":"hXqmEGunDz_q","outputId":"5ea3e348-3b33-43d3-9f52-2e27967490bb"},"outputs":[{"output_type":"stream","name":"stdout","text":["root\n"," |-- srcip: string (nullable = true)\n"," |-- sport: integer (nullable = true)\n"," |-- dstip: string (nullable = true)\n"," |-- dsport: integer (nullable = true)\n"," |-- proto: string (nullable = true)\n"," |-- state: string (nullable = true)\n"," |-- dur: double (nullable = true)\n"," |-- sbytes: integer (nullable = true)\n"," |-- dbytes: integer (nullable = true)\n"," |-- sttl: integer (nullable = true)\n"," |-- dttl: integer (nullable = true)\n"," |-- sloss: integer (nullable = true)\n"," |-- dloss: integer (nullable = true)\n"," |-- service: string (nullable = true)\n"," |-- Sload: double (nullable = true)\n"," |-- Dload: double (nullable = true)\n"," |-- Spkts: integer (nullable = true)\n"," |-- Dpkts: integer (nullable = true)\n"," |-- swin: integer (nullable = true)\n"," |-- dwin: integer (nullable = true)\n"," |-- stcpb: long (nullable = true)\n"," |-- dtcpb: long (nullable = true)\n"," |-- smeansz: integer (nullable = true)\n"," |-- dmeansz: integer (nullable = true)\n"," |-- trans_depth: integer (nullable = true)\n"," |-- res_bdy_len: integer (nullable = true)\n"," |-- Sjit: double (nullable = true)\n"," |-- Djit: double (nullable = true)\n"," |-- Stime: integer (nullable = true)\n"," |-- Ltime: integer (nullable = true)\n"," |-- Spkt: double (nullable = true)\n"," |-- Dpkt: double (nullable = true)\n"," |-- tcprtt: double (nullable = true)\n"," |-- synack: double (nullable = true)\n"," |-- ackdat: double (nullable = true)\n"," |-- is_sm_ips_ports: integer (nullable = true)\n"," |-- ct_state_ttl: integer (nullable = true)\n"," |-- ct_flw_http_mthd: integer (nullable = true)\n"," |-- is_ftp_login: integer (nullable = true)\n"," |-- ct_ftp_cmd: integer (nullable = true)\n"," |-- ct_srv_src: integer (nullable = true)\n"," |-- ct_srv_dst: integer (nullable = true)\n"," |-- ct_dst_ltm: integer (nullable = true)\n"," |-- ct_src_ ltm: integer (nullable = true)\n"," |-- ct_src_dport_ltm: integer (nullable = true)\n"," |-- ct_dst_sport_ltm: integer (nullable = true)\n"," |-- ct_dst_src_ltm: integer (nullable = true)\n"," |-- attack_cat: string (nullable = true)\n"," |-- Label: integer (nullable = true)\n","\n"]}],"source":["#Display Schema\n","UNSWNB15.printSchema()"]},{"cell_type":"code","execution_count":9,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":1597,"status":"ok","timestamp":1648913016254,"user":{"displayName":"Abdulwahab Abdulliameed","userId":"10732377996348789858"},"user_tz":-60},"id":"FzsO-MGXzOx4","outputId":"29479278-9662-4ba6-edac-59429030ae6b"},"outputs":[{"output_type":"stream","name":"stdout","text":["+----------+-----+-------------+------+-----+-----+------------+------+------+----+----+-----+-----+--------+---------+---------+-----+-----+----+----+----------+----------+-------+-------+-----------+-----------+----------+---------+----------+----------+------------+------------+------------+------------+------------+---------------+------------+----------------+------------+----------+----------+----------+----------+-----------+----------------+----------------+--------------+----------+-----+\n","|     srcip|sport|        dstip|dsport|proto|state|         dur|sbytes|dbytes|sttl|dttl|sloss|dloss| service|    Sload|    Dload|Spkts|Dpkts|swin|dwin|     stcpb|     dtcpb|smeansz|dmeansz|trans_depth|res_bdy_len|      Sjit|     Djit|     Stime|     Ltime|        Spkt|        Dpkt|      tcprtt|      synack|      ackdat|is_sm_ips_ports|ct_state_ttl|ct_flw_http_mthd|is_ftp_login|ct_ftp_cmd|ct_srv_src|ct_srv_dst|ct_dst_ltm|ct_src_ ltm|ct_src_dport_ltm|ct_dst_sport_ltm|ct_dst_src_ltm|attack_cat|Label|\n","+----------+-----+-------------+------+-----+-----+------------+------+------+----+----+-----+-----+--------+---------+---------+-----+-----+----+----+----------+----------+-------+-------+-----------+-----------+----------+---------+----------+----------+------------+------------+------------+------------+------------+---------------+------------+----------------+------------+----------+----------+----------+----------+-----------+----------------+----------------+--------------+----------+-----+\n","|59.166.0.3|56716|149.171.126.8|   143|  tcp|  FIN|  0.82546002|  7812| 16236|  31|  29|   30|   32|       -| 75090.25|156111.73|  122|  126| 255| 255|2751097753|2748686736|     64|    129|          0|          0| 445.25928| 474.9451|1421970774|1421970775|   6.8190908|    6.599896|5.9700001E-4|4.6899999E-4|     1.28E-4|              0|           0|               0|           0|         0|         2|         7|         1|          4|               1|               1|             1|      null|    0|\n","|59.166.0.0|43467|149.171.126.6| 49729|  tcp|  FIN|    0.101815|  4238| 65628|  31|  29|    7|   30|       -|328438.84|5087030.5|   72|   74| 255| 255| 961515433|3225510659|     59|    887|          0|          0|       0.0|91.579567|1421970775|1421970775|    1.429493|    1.387192|      6.8E-4|5.4600002E-4|     1.34E-4|              0|           0|               0|           0|         0|         7|         4|         1|          6|               1|               1|             1|      null|    0|\n","|59.166.0.5|41289|149.171.126.2|  9574|  tcp|  FIN| 0.044002999|  2750| 29104|  31|  29|    7|   17|       -|488693.97|5181101.5|   44|   48| 255| 255|3291096757|1191410228|     63|    606|          0|          0| 78.126968|62.206562|1421970775|1421970775|    1.014977|  0.92583001|     0.00125|     4.85E-4|     7.65E-4|              0|           0|               0|           0|         0|         3|         5|         3|          3|               1|               1|             1|      null|    0|\n","|59.166.0.9|43785|149.171.126.0|  6881|  tcp|  FIN|   2.7908299| 10476|395734|  31|  29|   16|  143|       -|29863.518|1130840.8|  180|  320| 255| 255|3934392726|3961690324|     58|   1237|          0|          0| 2707.4927| 2018.976|1421970772|1421970775|   15.589459|   8.7470121|6.8400003E-4|5.3199998E-4|1.5199999E-4|              0|           0|               0|           0|         0|        11|         4|         3|          2|               1|               1|             1|      null|    0|\n","|59.166.0.8|40691|149.171.126.9|  6881|  tcp|  FIN|   2.6335001| 13350|548216|  31|  29|   21|  197|       -|40381.238|1661560.6|  232|  438| 255| 255|   1518931|  18267719|     58|   1252|          0|          0| 718.33679|500.57288|1421970773|1421970775|   11.399026|   6.0251832|     6.19E-4|     4.89E-4|      1.3E-4|              0|           0|               0|           0|         0|        16|         7|         7|          1|               1|               1|             1|      null|    0|\n","|59.166.0.3|20393|149.171.126.3|  5190|  tcp|  FIN|    0.115048|  1958|  2308|  31|  29|    6|    6|       -|129963.15|153814.06|   22|   24| 255| 255|3646899201|3651364285|     89|     96|          0|          0| 435.26627|417.08563|1421970775|1421970775|    5.460381|    4.976913|7.0999999E-4|     5.73E-4|     1.37E-4|              0|           0|               0|           0|         0|         2|         6|         1|          4|               1|               1|             1|      null|    0|\n","|59.166.0.7|19792|149.171.126.0|    53|  udp|  CON|    0.003362|   146|   178|  31|  29|    0|    0|     dns|173706.13| 211778.7|    2|    2|   0|   0|         0|         0|     73|     89|          0|          0|       0.0|      0.0|1421970775|1421970775|       0.011|0.0060000001|         0.0|         0.0|         0.0|              0|           0|               0|           0|         0|         3|         2|         3|          3|               3|               1|             1|      null|    0|\n","|59.166.0.3|14382|149.171.126.9|  3354|  tcp|  FIN|  0.45305201|   424|  8824|  31|  29|    1|    4|ftp-data| 6551.124| 142835.7|    8|   12| 255| 255|2206905053|3307670308|     53|    735|          0|          0| 3906.7949|3074.6694|1421970775|1421970775|   64.671288|   41.134998|      6.8E-4|5.5900001E-4|     1.21E-4|              0|           0|               0|           0|         0|         4|         6|         7|          4|               1|               1|             2|      null|    0|\n","|59.166.0.9|37074|149.171.126.2|    53|  udp|  CON|    0.001088|   146|   178|  31|  29|    0|    0|     dns|536764.69|654411.75|    2|    2|   0|   0|         0|         0|     73|     89|          0|          0|       0.0|      0.0|1421970775|1421970775|       0.001|0.0089999996|         0.0|         0.0|         0.0|              0|           0|               0|           0|         0|         2|         5|         3|          2|               1|               1|             1|      null|    0|\n","|59.166.0.7|12569|149.171.126.5|    53|  udp|  CON|9.6899999E-4|   146|   178|  31|  29|    0|    0|     dns|602683.19|734778.13|    2|    2|   0|   0|         0|         0|     73|     89|          0|          0|       0.0|      0.0|1421970775|1421970775|0.0099999998|       0.003|         0.0|         0.0|         0.0|              0|           0|               0|           0|         0|         3|         1|         2|          3|               3|               1|             1|      null|    0|\n","|59.166.0.1|12792|149.171.126.7|    53|  udp|  CON|0.0010629999|   146|   178|  31|  29|    0|    0|     dns|549388.56| 669802.5|    2|    2|   0|   0|         0|         0|     73|     89|          0|          0|       0.0|      0.0|1421970780|1421970780|0.0020000001|       0.003|         0.0|         0.0|         0.0|              0|           0|               0|           0|         0|         5|         3|         2|          4|               3|               1|             1|      null|    0|\n","|59.166.0.0|63414|149.171.126.9| 10330|  tcp|  FIN|  0.26501101|  8928|   320|  31|  29|    4|    1|ftp-data|250283.94|8060.0425|   14|    6| 255| 255|2386904726|3699988372|    638|     53|          0|          0| 1725.2916|71.464249|1421970775|1421970776|   20.385462|   52.644802|7.7500002E-4|6.3299999E-4|     1.42E-4|              0|           0|               0|           0|         0|         3|         6|         6|          5|               1|               1|             2|      null|    0|\n","|59.166.0.1|33555|149.171.126.3|  6881|  tcp|  FIN|  0.51712799|  1540|  1644|  31|  29|    4|    4|       -|22338.764|  24025.0|   16|   18| 255| 255|1741520309|3943579644|     96|     91|          0|          0| 2036.1301| 51.91766|1421970776|1421970776|   34.433331|   30.388353|6.5399997E-4|5.2200002E-4|     1.32E-4|              0|           0|               0|           0|         0|         4|         6|         5|          7|               4|               1|             4|      null|    0|\n","|59.166.0.8|10867|149.171.126.8|   111|  udp|  CON|0.0053389999|   568|   312|  31|  29|    0|    0|       -|638321.81|350627.47|    4|    4|   0|   0|         0|         0|    142|     78|          0|          0| 1.7430201| 1.757632|1421970776|1421970776|        1.24|    1.252333|         0.0|         0.0|         0.0|              0|           0|               0|           0|         0|        16|         7|         5|          5|               1|               1|             4|      null|    0|\n","|59.166.0.8|12411|149.171.126.8|  1715|  udp|  CON|    0.001739|   512|   304|  31|  29|    0|    0|       -|1766532.5|1048878.6|    4|    4|   0|   0|         0|         0|    128|     76|          0|          0|0.63026798| 0.318434|1421970776|1421970776|  0.44966701|    0.227667|         0.0|         0.0|         0.0|              0|           0|               0|           0|         0|        16|         7|         5|          5|               1|               1|             4|      null|    0|\n","|59.166.0.8|46725|149.171.126.2|    53|  udp|  CON|    0.001018|   146|   178|  31|  29|    0|    0|     dns|573673.88|699410.63|    2|    2|   0|   0|         0|         0|     73|     89|          0|          0|       0.0|      0.0|1421970776|1421970776|       0.011|       0.011|         0.0|         0.0|         0.0|              0|           0|               0|           0|         0|         2|         5|         3|          5|               1|               1|             1|      null|    0|\n","|59.166.0.1|51562|149.171.126.4|    53|  udp|  CON|    0.001044|   146|   178|  31|  29|    0|    0|     dns|559386.94|681992.31|    2|    2|   0|   0|         0|         0|     73|     89|          0|          0|       0.0|      0.0|1421970776|1421970776|       0.011|       0.003|         0.0|         0.0|         0.0|              0|           0|               0|           0|         0|         2|         3|         5|          7|               2|               1|             3|      null|    0|\n","|59.166.0.3|48838|149.171.126.2|    53|  udp|  CON|9.8699995E-4|   146|   178|  31|  29|    0|    0|     dns| 591692.0|721377.94|    2|    2|   0|   0|         0|         0|     73|     89|          0|          0|       0.0|      0.0|1421970776|1421970776|0.0080000004|       0.003|         0.0|         0.0|         0.0|              0|           0|               0|           0|         0|         2|         5|         3|          3|               1|               1|             1|      null|    0|\n","|59.166.0.0|16907|149.171.126.9|    21|  tcp|  FIN|   2.2547121|  2934|  3740|  31|  29|   11|   15|     ftp|10211.503|13025.166|   52|   54| 255| 255| 241515551|2584135680|     56|     69|          0|          0| 3141.3708| 99.93412|1421970773|1421970776|   44.203648|   42.531734|6.6100003E-4|5.2499998E-4|     1.36E-4|              0|           0|               0|           0|         0|         1|         3|         6|          5|               1|               1|             2|      null|    0|\n","|59.166.0.0| 1915|149.171.126.4| 32945|  tcp|  FIN| 0.051220998|  2854| 29104|  31|  29|    7|   17|       -|436071.16|4450987.0|   46|   48| 255| 255|1921515932|1974066994|     62|    606|          0|          0| 78.226654|76.387978|1421970776|1421970776|   1.1294219|   1.0781699|      8.1E-4|     5.43E-4|     2.67E-4|              0|           0|               0|           0|         0|         7|         2|         5|          5|               1|               1|             2|      null|    0|\n","+----------+-----+-------------+------+-----+-----+------------+------+------+----+----+-----+-----+--------+---------+---------+-----+-----+----+----+----------+----------+-------+-------+-----------+-----------+----------+---------+----------+----------+------------+------------+------------+------------+------------+---------------+------------+----------------+------------+----------+----------+----------+----------+-----------+----------------+----------------+--------------+----------+-----+\n","only showing top 20 rows\n","\n"]}],"source":["#Show Dataframe UNSWNB15\n","UNSWNB15.show()"]},{"cell_type":"code","source":["#Replace null values in attack_cat column with normal\n","dataset = UNSWNB15.na.fill(value='normal',subset=[\"attack_cat\"])\n","dataset.show()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"H_6Ei-0yjGw8","executionInfo":{"status":"ok","timestamp":1648913021266,"user_tz":-60,"elapsed":915,"user":{"displayName":"Abdulwahab Abdulliameed","userId":"10732377996348789858"}},"outputId":"94cf5c9e-8653-4613-ac8b-30bda8bf5f5c"},"execution_count":10,"outputs":[{"output_type":"stream","name":"stdout","text":["+----------+-----+-------------+------+-----+-----+------------+------+------+----+----+-----+-----+--------+---------+---------+-----+-----+----+----+----------+----------+-------+-------+-----------+-----------+----------+---------+----------+----------+------------+------------+------------+------------+------------+---------------+------------+----------------+------------+----------+----------+----------+----------+-----------+----------------+----------------+--------------+----------+-----+\n","|     srcip|sport|        dstip|dsport|proto|state|         dur|sbytes|dbytes|sttl|dttl|sloss|dloss| service|    Sload|    Dload|Spkts|Dpkts|swin|dwin|     stcpb|     dtcpb|smeansz|dmeansz|trans_depth|res_bdy_len|      Sjit|     Djit|     Stime|     Ltime|        Spkt|        Dpkt|      tcprtt|      synack|      ackdat|is_sm_ips_ports|ct_state_ttl|ct_flw_http_mthd|is_ftp_login|ct_ftp_cmd|ct_srv_src|ct_srv_dst|ct_dst_ltm|ct_src_ ltm|ct_src_dport_ltm|ct_dst_sport_ltm|ct_dst_src_ltm|attack_cat|Label|\n","+----------+-----+-------------+------+-----+-----+------------+------+------+----+----+-----+-----+--------+---------+---------+-----+-----+----+----+----------+----------+-------+-------+-----------+-----------+----------+---------+----------+----------+------------+------------+------------+------------+------------+---------------+------------+----------------+------------+----------+----------+----------+----------+-----------+----------------+----------------+--------------+----------+-----+\n","|59.166.0.3|56716|149.171.126.8|   143|  tcp|  FIN|  0.82546002|  7812| 16236|  31|  29|   30|   32|       -| 75090.25|156111.73|  122|  126| 255| 255|2751097753|2748686736|     64|    129|          0|          0| 445.25928| 474.9451|1421970774|1421970775|   6.8190908|    6.599896|5.9700001E-4|4.6899999E-4|     1.28E-4|              0|           0|               0|           0|         0|         2|         7|         1|          4|               1|               1|             1|    normal|    0|\n","|59.166.0.0|43467|149.171.126.6| 49729|  tcp|  FIN|    0.101815|  4238| 65628|  31|  29|    7|   30|       -|328438.84|5087030.5|   72|   74| 255| 255| 961515433|3225510659|     59|    887|          0|          0|       0.0|91.579567|1421970775|1421970775|    1.429493|    1.387192|      6.8E-4|5.4600002E-4|     1.34E-4|              0|           0|               0|           0|         0|         7|         4|         1|          6|               1|               1|             1|    normal|    0|\n","|59.166.0.5|41289|149.171.126.2|  9574|  tcp|  FIN| 0.044002999|  2750| 29104|  31|  29|    7|   17|       -|488693.97|5181101.5|   44|   48| 255| 255|3291096757|1191410228|     63|    606|          0|          0| 78.126968|62.206562|1421970775|1421970775|    1.014977|  0.92583001|     0.00125|     4.85E-4|     7.65E-4|              0|           0|               0|           0|         0|         3|         5|         3|          3|               1|               1|             1|    normal|    0|\n","|59.166.0.9|43785|149.171.126.0|  6881|  tcp|  FIN|   2.7908299| 10476|395734|  31|  29|   16|  143|       -|29863.518|1130840.8|  180|  320| 255| 255|3934392726|3961690324|     58|   1237|          0|          0| 2707.4927| 2018.976|1421970772|1421970775|   15.589459|   8.7470121|6.8400003E-4|5.3199998E-4|1.5199999E-4|              0|           0|               0|           0|         0|        11|         4|         3|          2|               1|               1|             1|    normal|    0|\n","|59.166.0.8|40691|149.171.126.9|  6881|  tcp|  FIN|   2.6335001| 13350|548216|  31|  29|   21|  197|       -|40381.238|1661560.6|  232|  438| 255| 255|   1518931|  18267719|     58|   1252|          0|          0| 718.33679|500.57288|1421970773|1421970775|   11.399026|   6.0251832|     6.19E-4|     4.89E-4|      1.3E-4|              0|           0|               0|           0|         0|        16|         7|         7|          1|               1|               1|             1|    normal|    0|\n","|59.166.0.3|20393|149.171.126.3|  5190|  tcp|  FIN|    0.115048|  1958|  2308|  31|  29|    6|    6|       -|129963.15|153814.06|   22|   24| 255| 255|3646899201|3651364285|     89|     96|          0|          0| 435.26627|417.08563|1421970775|1421970775|    5.460381|    4.976913|7.0999999E-4|     5.73E-4|     1.37E-4|              0|           0|               0|           0|         0|         2|         6|         1|          4|               1|               1|             1|    normal|    0|\n","|59.166.0.7|19792|149.171.126.0|    53|  udp|  CON|    0.003362|   146|   178|  31|  29|    0|    0|     dns|173706.13| 211778.7|    2|    2|   0|   0|         0|         0|     73|     89|          0|          0|       0.0|      0.0|1421970775|1421970775|       0.011|0.0060000001|         0.0|         0.0|         0.0|              0|           0|               0|           0|         0|         3|         2|         3|          3|               3|               1|             1|    normal|    0|\n","|59.166.0.3|14382|149.171.126.9|  3354|  tcp|  FIN|  0.45305201|   424|  8824|  31|  29|    1|    4|ftp-data| 6551.124| 142835.7|    8|   12| 255| 255|2206905053|3307670308|     53|    735|          0|          0| 3906.7949|3074.6694|1421970775|1421970775|   64.671288|   41.134998|      6.8E-4|5.5900001E-4|     1.21E-4|              0|           0|               0|           0|         0|         4|         6|         7|          4|               1|               1|             2|    normal|    0|\n","|59.166.0.9|37074|149.171.126.2|    53|  udp|  CON|    0.001088|   146|   178|  31|  29|    0|    0|     dns|536764.69|654411.75|    2|    2|   0|   0|         0|         0|     73|     89|          0|          0|       0.0|      0.0|1421970775|1421970775|       0.001|0.0089999996|         0.0|         0.0|         0.0|              0|           0|               0|           0|         0|         2|         5|         3|          2|               1|               1|             1|    normal|    0|\n","|59.166.0.7|12569|149.171.126.5|    53|  udp|  CON|9.6899999E-4|   146|   178|  31|  29|    0|    0|     dns|602683.19|734778.13|    2|    2|   0|   0|         0|         0|     73|     89|          0|          0|       0.0|      0.0|1421970775|1421970775|0.0099999998|       0.003|         0.0|         0.0|         0.0|              0|           0|               0|           0|         0|         3|         1|         2|          3|               3|               1|             1|    normal|    0|\n","|59.166.0.1|12792|149.171.126.7|    53|  udp|  CON|0.0010629999|   146|   178|  31|  29|    0|    0|     dns|549388.56| 669802.5|    2|    2|   0|   0|         0|         0|     73|     89|          0|          0|       0.0|      0.0|1421970780|1421970780|0.0020000001|       0.003|         0.0|         0.0|         0.0|              0|           0|               0|           0|         0|         5|         3|         2|          4|               3|               1|             1|    normal|    0|\n","|59.166.0.0|63414|149.171.126.9| 10330|  tcp|  FIN|  0.26501101|  8928|   320|  31|  29|    4|    1|ftp-data|250283.94|8060.0425|   14|    6| 255| 255|2386904726|3699988372|    638|     53|          0|          0| 1725.2916|71.464249|1421970775|1421970776|   20.385462|   52.644802|7.7500002E-4|6.3299999E-4|     1.42E-4|              0|           0|               0|           0|         0|         3|         6|         6|          5|               1|               1|             2|    normal|    0|\n","|59.166.0.1|33555|149.171.126.3|  6881|  tcp|  FIN|  0.51712799|  1540|  1644|  31|  29|    4|    4|       -|22338.764|  24025.0|   16|   18| 255| 255|1741520309|3943579644|     96|     91|          0|          0| 2036.1301| 51.91766|1421970776|1421970776|   34.433331|   30.388353|6.5399997E-4|5.2200002E-4|     1.32E-4|              0|           0|               0|           0|         0|         4|         6|         5|          7|               4|               1|             4|    normal|    0|\n","|59.166.0.8|10867|149.171.126.8|   111|  udp|  CON|0.0053389999|   568|   312|  31|  29|    0|    0|       -|638321.81|350627.47|    4|    4|   0|   0|         0|         0|    142|     78|          0|          0| 1.7430201| 1.757632|1421970776|1421970776|        1.24|    1.252333|         0.0|         0.0|         0.0|              0|           0|               0|           0|         0|        16|         7|         5|          5|               1|               1|             4|    normal|    0|\n","|59.166.0.8|12411|149.171.126.8|  1715|  udp|  CON|    0.001739|   512|   304|  31|  29|    0|    0|       -|1766532.5|1048878.6|    4|    4|   0|   0|         0|         0|    128|     76|          0|          0|0.63026798| 0.318434|1421970776|1421970776|  0.44966701|    0.227667|         0.0|         0.0|         0.0|              0|           0|               0|           0|         0|        16|         7|         5|          5|               1|               1|             4|    normal|    0|\n","|59.166.0.8|46725|149.171.126.2|    53|  udp|  CON|    0.001018|   146|   178|  31|  29|    0|    0|     dns|573673.88|699410.63|    2|    2|   0|   0|         0|         0|     73|     89|          0|          0|       0.0|      0.0|1421970776|1421970776|       0.011|       0.011|         0.0|         0.0|         0.0|              0|           0|               0|           0|         0|         2|         5|         3|          5|               1|               1|             1|    normal|    0|\n","|59.166.0.1|51562|149.171.126.4|    53|  udp|  CON|    0.001044|   146|   178|  31|  29|    0|    0|     dns|559386.94|681992.31|    2|    2|   0|   0|         0|         0|     73|     89|          0|          0|       0.0|      0.0|1421970776|1421970776|       0.011|       0.003|         0.0|         0.0|         0.0|              0|           0|               0|           0|         0|         2|         3|         5|          7|               2|               1|             3|    normal|    0|\n","|59.166.0.3|48838|149.171.126.2|    53|  udp|  CON|9.8699995E-4|   146|   178|  31|  29|    0|    0|     dns| 591692.0|721377.94|    2|    2|   0|   0|         0|         0|     73|     89|          0|          0|       0.0|      0.0|1421970776|1421970776|0.0080000004|       0.003|         0.0|         0.0|         0.0|              0|           0|               0|           0|         0|         2|         5|         3|          3|               1|               1|             1|    normal|    0|\n","|59.166.0.0|16907|149.171.126.9|    21|  tcp|  FIN|   2.2547121|  2934|  3740|  31|  29|   11|   15|     ftp|10211.503|13025.166|   52|   54| 255| 255| 241515551|2584135680|     56|     69|          0|          0| 3141.3708| 99.93412|1421970773|1421970776|   44.203648|   42.531734|6.6100003E-4|5.2499998E-4|     1.36E-4|              0|           0|               0|           0|         0|         1|         3|         6|          5|               1|               1|             2|    normal|    0|\n","|59.166.0.0| 1915|149.171.126.4| 32945|  tcp|  FIN| 0.051220998|  2854| 29104|  31|  29|    7|   17|       -|436071.16|4450987.0|   46|   48| 255| 255|1921515932|1974066994|     62|    606|          0|          0| 78.226654|76.387978|1421970776|1421970776|   1.1294219|   1.0781699|      8.1E-4|     5.43E-4|     2.67E-4|              0|           0|               0|           0|         0|         7|         2|         5|          5|               1|               1|             2|    normal|    0|\n","+----------+-----+-------------+------+-----+-----+------------+------+------+----+----+-----+-----+--------+---------+---------+-----+-----+----+----+----------+----------+-------+-------+-----------+-----------+----------+---------+----------+----------+------------+------------+------------+------------+------------+---------------+------------+----------------+------------+----------+----------+----------+----------+-----------+----------------+----------------+--------------+----------+-----+\n","only showing top 20 rows\n","\n"]}]},{"cell_type":"code","source":["#Update all values in attack_cat column\n","#from pyspark.sql.functions import col,when\n","from pyspark.sql.functions import col,when\n","updated_dataset = dataset.withColumn(\"attack_cat\", when( col(\"attack_cat\")==\"normal\",\"Normal\" )\\\n",".when( col(\"attack_cat\")==\"Generic\",\"Generic\" )\\\n",".when( col(\"attack_cat\")==\"Exploits\",\"Exploits\" )\\\n",".when( col(\"attack_cat\")==\"DoS\",\"DoS\" )\\\n",".when( col(\"attack_cat\")==\"Fuzzers\",\"Fuzzers\" )\\\n",".when( col(\"attack_cat\")==\"Analysis\",\"Analysis\" )\\\n",".when( col(\"attack_cat\")==\"Reconnaissance\",\"Reconnaissance\" )\\\n",".when( col(\"attack_cat\")==\"Backdoors\",\"Backdoors\" )\\\n",".when( col(\"attack_cat\")==\"Shellcode\",\"Shellcode\" )\\\n",".when( col(\"attack_cat\")==\"Worms\",\"Worms\" )\\\n",".when( col(\"attack_cat\")==\"Fuzzers \", \"Fuzzers\")\\\n",".when( col(\"attack_cat\")==\"Reconnaissance \",\"Reconnaissance\" )\\\n",".when( col(\"attack_cat\")==\"Backdoor\",\"Backdoors\" )\\\n",".when( col(\"attack_cat\")==\"Shellcode \",\"Shellcode\" )\\\n",")"],"metadata":{"id":"YjGrQpha4B_U","executionInfo":{"status":"ok","timestamp":1648913111325,"user_tz":-60,"elapsed":915,"user":{"displayName":"Abdulwahab Abdulliameed","userId":"10732377996348789858"}}},"execution_count":11,"outputs":[]},{"cell_type":"code","source":["#Confirm the dataset and the number of records\n","updated_dataset.show()\n","updated_dataset.count()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"WfNwYKBv4fOC","executionInfo":{"status":"ok","timestamp":1648913330662,"user_tz":-60,"elapsed":5739,"user":{"displayName":"Abdulwahab Abdulliameed","userId":"10732377996348789858"}},"outputId":"102864bb-e498-455c-a950-8c4eb4d3abcb"},"execution_count":12,"outputs":[{"output_type":"stream","name":"stdout","text":["+----------+-----+-------------+------+-----+-----+------------+------+------+----+----+-----+-----+--------+---------+---------+-----+-----+----+----+----------+----------+-------+-------+-----------+-----------+----------+---------+----------+----------+------------+------------+------------+------------+------------+---------------+------------+----------------+------------+----------+----------+----------+----------+-----------+----------------+----------------+--------------+----------+-----+\n","|     srcip|sport|        dstip|dsport|proto|state|         dur|sbytes|dbytes|sttl|dttl|sloss|dloss| service|    Sload|    Dload|Spkts|Dpkts|swin|dwin|     stcpb|     dtcpb|smeansz|dmeansz|trans_depth|res_bdy_len|      Sjit|     Djit|     Stime|     Ltime|        Spkt|        Dpkt|      tcprtt|      synack|      ackdat|is_sm_ips_ports|ct_state_ttl|ct_flw_http_mthd|is_ftp_login|ct_ftp_cmd|ct_srv_src|ct_srv_dst|ct_dst_ltm|ct_src_ ltm|ct_src_dport_ltm|ct_dst_sport_ltm|ct_dst_src_ltm|attack_cat|Label|\n","+----------+-----+-------------+------+-----+-----+------------+------+------+----+----+-----+-----+--------+---------+---------+-----+-----+----+----+----------+----------+-------+-------+-----------+-----------+----------+---------+----------+----------+------------+------------+------------+------------+------------+---------------+------------+----------------+------------+----------+----------+----------+----------+-----------+----------------+----------------+--------------+----------+-----+\n","|59.166.0.3|56716|149.171.126.8|   143|  tcp|  FIN|  0.82546002|  7812| 16236|  31|  29|   30|   32|       -| 75090.25|156111.73|  122|  126| 255| 255|2751097753|2748686736|     64|    129|          0|          0| 445.25928| 474.9451|1421970774|1421970775|   6.8190908|    6.599896|5.9700001E-4|4.6899999E-4|     1.28E-4|              0|           0|               0|           0|         0|         2|         7|         1|          4|               1|               1|             1|    Normal|    0|\n","|59.166.0.0|43467|149.171.126.6| 49729|  tcp|  FIN|    0.101815|  4238| 65628|  31|  29|    7|   30|       -|328438.84|5087030.5|   72|   74| 255| 255| 961515433|3225510659|     59|    887|          0|          0|       0.0|91.579567|1421970775|1421970775|    1.429493|    1.387192|      6.8E-4|5.4600002E-4|     1.34E-4|              0|           0|               0|           0|         0|         7|         4|         1|          6|               1|               1|             1|    Normal|    0|\n","|59.166.0.5|41289|149.171.126.2|  9574|  tcp|  FIN| 0.044002999|  2750| 29104|  31|  29|    7|   17|       -|488693.97|5181101.5|   44|   48| 255| 255|3291096757|1191410228|     63|    606|          0|          0| 78.126968|62.206562|1421970775|1421970775|    1.014977|  0.92583001|     0.00125|     4.85E-4|     7.65E-4|              0|           0|               0|           0|         0|         3|         5|         3|          3|               1|               1|             1|    Normal|    0|\n","|59.166.0.9|43785|149.171.126.0|  6881|  tcp|  FIN|   2.7908299| 10476|395734|  31|  29|   16|  143|       -|29863.518|1130840.8|  180|  320| 255| 255|3934392726|3961690324|     58|   1237|          0|          0| 2707.4927| 2018.976|1421970772|1421970775|   15.589459|   8.7470121|6.8400003E-4|5.3199998E-4|1.5199999E-4|              0|           0|               0|           0|         0|        11|         4|         3|          2|               1|               1|             1|    Normal|    0|\n","|59.166.0.8|40691|149.171.126.9|  6881|  tcp|  FIN|   2.6335001| 13350|548216|  31|  29|   21|  197|       -|40381.238|1661560.6|  232|  438| 255| 255|   1518931|  18267719|     58|   1252|          0|          0| 718.33679|500.57288|1421970773|1421970775|   11.399026|   6.0251832|     6.19E-4|     4.89E-4|      1.3E-4|              0|           0|               0|           0|         0|        16|         7|         7|          1|               1|               1|             1|    Normal|    0|\n","|59.166.0.3|20393|149.171.126.3|  5190|  tcp|  FIN|    0.115048|  1958|  2308|  31|  29|    6|    6|       -|129963.15|153814.06|   22|   24| 255| 255|3646899201|3651364285|     89|     96|          0|          0| 435.26627|417.08563|1421970775|1421970775|    5.460381|    4.976913|7.0999999E-4|     5.73E-4|     1.37E-4|              0|           0|               0|           0|         0|         2|         6|         1|          4|               1|               1|             1|    Normal|    0|\n","|59.166.0.7|19792|149.171.126.0|    53|  udp|  CON|    0.003362|   146|   178|  31|  29|    0|    0|     dns|173706.13| 211778.7|    2|    2|   0|   0|         0|         0|     73|     89|          0|          0|       0.0|      0.0|1421970775|1421970775|       0.011|0.0060000001|         0.0|         0.0|         0.0|              0|           0|               0|           0|         0|         3|         2|         3|          3|               3|               1|             1|    Normal|    0|\n","|59.166.0.3|14382|149.171.126.9|  3354|  tcp|  FIN|  0.45305201|   424|  8824|  31|  29|    1|    4|ftp-data| 6551.124| 142835.7|    8|   12| 255| 255|2206905053|3307670308|     53|    735|          0|          0| 3906.7949|3074.6694|1421970775|1421970775|   64.671288|   41.134998|      6.8E-4|5.5900001E-4|     1.21E-4|              0|           0|               0|           0|         0|         4|         6|         7|          4|               1|               1|             2|    Normal|    0|\n","|59.166.0.9|37074|149.171.126.2|    53|  udp|  CON|    0.001088|   146|   178|  31|  29|    0|    0|     dns|536764.69|654411.75|    2|    2|   0|   0|         0|         0|     73|     89|          0|          0|       0.0|      0.0|1421970775|1421970775|       0.001|0.0089999996|         0.0|         0.0|         0.0|              0|           0|               0|           0|         0|         2|         5|         3|          2|               1|               1|             1|    Normal|    0|\n","|59.166.0.7|12569|149.171.126.5|    53|  udp|  CON|9.6899999E-4|   146|   178|  31|  29|    0|    0|     dns|602683.19|734778.13|    2|    2|   0|   0|         0|         0|     73|     89|          0|          0|       0.0|      0.0|1421970775|1421970775|0.0099999998|       0.003|         0.0|         0.0|         0.0|              0|           0|               0|           0|         0|         3|         1|         2|          3|               3|               1|             1|    Normal|    0|\n","|59.166.0.1|12792|149.171.126.7|    53|  udp|  CON|0.0010629999|   146|   178|  31|  29|    0|    0|     dns|549388.56| 669802.5|    2|    2|   0|   0|         0|         0|     73|     89|          0|          0|       0.0|      0.0|1421970780|1421970780|0.0020000001|       0.003|         0.0|         0.0|         0.0|              0|           0|               0|           0|         0|         5|         3|         2|          4|               3|               1|             1|    Normal|    0|\n","|59.166.0.0|63414|149.171.126.9| 10330|  tcp|  FIN|  0.26501101|  8928|   320|  31|  29|    4|    1|ftp-data|250283.94|8060.0425|   14|    6| 255| 255|2386904726|3699988372|    638|     53|          0|          0| 1725.2916|71.464249|1421970775|1421970776|   20.385462|   52.644802|7.7500002E-4|6.3299999E-4|     1.42E-4|              0|           0|               0|           0|         0|         3|         6|         6|          5|               1|               1|             2|    Normal|    0|\n","|59.166.0.1|33555|149.171.126.3|  6881|  tcp|  FIN|  0.51712799|  1540|  1644|  31|  29|    4|    4|       -|22338.764|  24025.0|   16|   18| 255| 255|1741520309|3943579644|     96|     91|          0|          0| 2036.1301| 51.91766|1421970776|1421970776|   34.433331|   30.388353|6.5399997E-4|5.2200002E-4|     1.32E-4|              0|           0|               0|           0|         0|         4|         6|         5|          7|               4|               1|             4|    Normal|    0|\n","|59.166.0.8|10867|149.171.126.8|   111|  udp|  CON|0.0053389999|   568|   312|  31|  29|    0|    0|       -|638321.81|350627.47|    4|    4|   0|   0|         0|         0|    142|     78|          0|          0| 1.7430201| 1.757632|1421970776|1421970776|        1.24|    1.252333|         0.0|         0.0|         0.0|              0|           0|               0|           0|         0|        16|         7|         5|          5|               1|               1|             4|    Normal|    0|\n","|59.166.0.8|12411|149.171.126.8|  1715|  udp|  CON|    0.001739|   512|   304|  31|  29|    0|    0|       -|1766532.5|1048878.6|    4|    4|   0|   0|         0|         0|    128|     76|          0|          0|0.63026798| 0.318434|1421970776|1421970776|  0.44966701|    0.227667|         0.0|         0.0|         0.0|              0|           0|               0|           0|         0|        16|         7|         5|          5|               1|               1|             4|    Normal|    0|\n","|59.166.0.8|46725|149.171.126.2|    53|  udp|  CON|    0.001018|   146|   178|  31|  29|    0|    0|     dns|573673.88|699410.63|    2|    2|   0|   0|         0|         0|     73|     89|          0|          0|       0.0|      0.0|1421970776|1421970776|       0.011|       0.011|         0.0|         0.0|         0.0|              0|           0|               0|           0|         0|         2|         5|         3|          5|               1|               1|             1|    Normal|    0|\n","|59.166.0.1|51562|149.171.126.4|    53|  udp|  CON|    0.001044|   146|   178|  31|  29|    0|    0|     dns|559386.94|681992.31|    2|    2|   0|   0|         0|         0|     73|     89|          0|          0|       0.0|      0.0|1421970776|1421970776|       0.011|       0.003|         0.0|         0.0|         0.0|              0|           0|               0|           0|         0|         2|         3|         5|          7|               2|               1|             3|    Normal|    0|\n","|59.166.0.3|48838|149.171.126.2|    53|  udp|  CON|9.8699995E-4|   146|   178|  31|  29|    0|    0|     dns| 591692.0|721377.94|    2|    2|   0|   0|         0|         0|     73|     89|          0|          0|       0.0|      0.0|1421970776|1421970776|0.0080000004|       0.003|         0.0|         0.0|         0.0|              0|           0|               0|           0|         0|         2|         5|         3|          3|               1|               1|             1|    Normal|    0|\n","|59.166.0.0|16907|149.171.126.9|    21|  tcp|  FIN|   2.2547121|  2934|  3740|  31|  29|   11|   15|     ftp|10211.503|13025.166|   52|   54| 255| 255| 241515551|2584135680|     56|     69|          0|          0| 3141.3708| 99.93412|1421970773|1421970776|   44.203648|   42.531734|6.6100003E-4|5.2499998E-4|     1.36E-4|              0|           0|               0|           0|         0|         1|         3|         6|          5|               1|               1|             2|    Normal|    0|\n","|59.166.0.0| 1915|149.171.126.4| 32945|  tcp|  FIN| 0.051220998|  2854| 29104|  31|  29|    7|   17|       -|436071.16|4450987.0|   46|   48| 255| 255|1921515932|1974066994|     62|    606|          0|          0| 78.226654|76.387978|1421970776|1421970776|   1.1294219|   1.0781699|      8.1E-4|     5.43E-4|     2.67E-4|              0|           0|               0|           0|         0|         7|         2|         5|          5|               1|               1|             2|    Normal|    0|\n","+----------+-----+-------------+------+-----+-----+------------+------+------+----+----+-----+-----+--------+---------+---------+-----+-----+----+----+----------+----------+-------+-------+-----------+-----------+----------+---------+----------+----------+------------+------------+------------+------------+------------+---------------+------------+----------------+------------+----------+----------+----------+----------+-----------+----------------+----------------+--------------+----------+-----+\n","only showing top 20 rows\n","\n"]},{"output_type":"execute_result","data":{"text/plain":["2539739"]},"metadata":{},"execution_count":12}]},{"cell_type":"code","source":["#Randomly split data into training and test sets, and set seed for reproducibility.\n","trainDF, testDF = dataset.randomSplit([0.8, 0.2], seed=42)\n","print(trainDF.cache().count()) # Cache because accessing training data multiple times\n","print(testDF.count())"],"metadata":{"id":"W89cBWbiIRow","executionInfo":{"status":"ok","timestamp":1648913465420,"user_tz":-60,"elapsed":119465,"user":{"displayName":"Abdulwahab Abdulliameed","userId":"10732377996348789858"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"c9b7a342-a320-432e-d173-9c4edbd9a4cd"},"execution_count":13,"outputs":[{"output_type":"stream","name":"stdout","text":["2031752\n","507987\n"]}]},{"cell_type":"code","source":["#Import StringIndexer and OneHotEncoder\n","from pyspark.ml.feature import StringIndexer, OneHotEncoder\n","\n","categoricalCols = [\"srcip\", \"dstip\", \"proto\", \"state\"]\n","# The following two lines are estimators. They return functions that we will later apply to transform the dataset.\n","stringIndexer = StringIndexer(inputCols=categoricalCols, outputCols=[x + \"Index\" for x in categoricalCols]) \n","encoder = OneHotEncoder(inputCols=stringIndexer.getOutputCols(), outputCols=[x + \"OHE\" for x in categoricalCols]) \n","# The label column (\"attack_cat\") is also a string value \n","# Convert it to a numeric value using StringIndexer.\n","labelToIndex = StringIndexer(inputCol=\"attack_cat\", outputCol=\"label\")"],"metadata":{"id":"ZS8gImFTIWCs","executionInfo":{"status":"ok","timestamp":1648914447276,"user_tz":-60,"elapsed":350,"user":{"displayName":"Abdulwahab Abdulliameed","userId":"10732377996348789858"}}},"execution_count":14,"outputs":[]},{"cell_type":"code","source":["stringIndexerModel = stringIndexer.fit(trainDF)\n","display(stringIndexerModel.transform(trainDF))"],"metadata":{"id":"_y44v9hAGIxX","executionInfo":{"status":"ok","timestamp":1648914989681,"user_tz":-60,"elapsed":4340,"user":{"displayName":"Abdulwahab Abdulliameed","userId":"10732377996348789858"}},"colab":{"base_uri":"https://localhost:8080/","height":55},"outputId":"c4383519-55f3-488a-ec3c-a1875426daf7"},"execution_count":15,"outputs":[{"output_type":"display_data","data":{"text/plain":["DataFrame[srcip: string, sport: int, dstip: string, dsport: int, proto: string, state: string, dur: double, sbytes: int, dbytes: int, sttl: int, dttl: int, sloss: int, dloss: int, service: string, Sload: double, Dload: double, Spkts: int, Dpkts: int, swin: int, dwin: int, stcpb: bigint, dtcpb: bigint, smeansz: int, dmeansz: int, trans_depth: int, res_bdy_len: int, Sjit: double, Djit: double, Stime: int, Ltime: int, Spkt: double, Dpkt: double, tcprtt: double, synack: double, ackdat: double, is_sm_ips_ports: int, ct_state_ttl: int, ct_flw_http_mthd: int, is_ftp_login: int, ct_ftp_cmd: int, ct_srv_src: int, ct_srv_dst: int, ct_dst_ltm: int, ct_src_ ltm: int, ct_src_dport_ltm: int, ct_dst_sport_ltm: int, ct_dst_src_ltm: int, attack_cat: string, Label: int, srcipIndex: double, dstipIndex: double, protoIndex: double, stateIndex: double]"]},"metadata":{}}]},{"cell_type":"code","source":["#Import VectorAssembler\n","from pyspark.ml.feature import VectorAssembler\n","# This includes both the numeric columns and the one-hot encoded binary vector columns in the dataset.\n","numericCols = [\"sport\", \"dsport\", \"trans_depth\", \"dur\",\"Spkts\", \"Dpkts\"]\n","assemblerInputs = [c + \"OHE\" for c in categoricalCols] + numericCols\n","vecAssembler = VectorAssembler(inputCols=assemblerInputs, outputCol=\"features\")"],"metadata":{"id":"i0YAnRHbHMyY","executionInfo":{"status":"ok","timestamp":1648915355966,"user_tz":-60,"elapsed":369,"user":{"displayName":"Abdulwahab Abdulliameed","userId":"10732377996348789858"}}},"execution_count":16,"outputs":[]},{"cell_type":"code","source":["#Import LogisticRegression\n","from pyspark.ml.classification import LogisticRegression\n","#Model defination using LogisticRegression\n","lr = LogisticRegression(featuresCol=\"features\", labelCol=\"label\", regParam=1.0)"],"metadata":{"id":"5rlUU6hkK1xo","executionInfo":{"status":"ok","timestamp":1648915933298,"user_tz":-60,"elapsed":571,"user":{"displayName":"Abdulwahab Abdulliameed","userId":"10732377996348789858"}}},"execution_count":17,"outputs":[]},{"cell_type":"code","source":["#Import Pipeline\n","from pyspark.ml import Pipeline\n","# Define the pipeline based on the stages created in previous steps.\n","pipeline = Pipeline(stages=[stringIndexer, encoder, labelToIndex, vecAssembler, lr])\n","# Define the pipeline model.\n","pipelineModel = pipeline.fit(trainDF)\n","# Apply the pipeline model to the test dataset.\n","predDF = pipelineModel.transform(testDF)"],"metadata":{"id":"bSo_COCVK53m","executionInfo":{"status":"ok","timestamp":1648916616162,"user_tz":-60,"elapsed":195827,"user":{"displayName":"Abdulwahab Abdulliameed","userId":"10732377996348789858"}}},"execution_count":18,"outputs":[]},{"cell_type":"code","source":["#Display the predictions from the model. \n","display(predDF.select(\"features\", \"label\", \"prediction\", \"probability\"))\n","predDF.select(\"features\", \"label\", \"prediction\", \"probability\").show()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":499},"id":"n3ZEV-6iHGoi","executionInfo":{"status":"ok","timestamp":1648917152858,"user_tz":-60,"elapsed":9704,"user":{"displayName":"Abdulwahab Abdulliameed","userId":"10732377996348789858"}},"outputId":"57a67e76-f988-4368-950b-1a5d8c810479"},"execution_count":23,"outputs":[{"output_type":"display_data","data":{"text/plain":["DataFrame[features: vector, label: double, prediction: double, probability: vector]"]},"metadata":{}},{"output_type":"stream","name":"stdout","text":["+--------------------+-----+----------+--------------------+\n","|            features|label|prediction|         probability|\n","+--------------------+-----+----------+--------------------+\n","|(242,[24,69,91,22...|  0.0|       0.0|[0.89002597852031...|\n","|(242,[24,69,91,22...|  0.0|       0.0|[0.89002597852031...|\n","|(242,[24,69,91,22...|  0.0|       0.0|[0.89002597852031...|\n","|(242,[24,69,91,22...|  0.0|       0.0|[0.89002597852031...|\n","|(242,[24,69,91,22...|  0.0|       0.0|[0.89002597852031...|\n","|(242,[24,69,91,22...|  0.0|       0.0|[0.89002597852031...|\n","|(242,[24,69,91,22...|  0.0|       0.0|[0.89002597852031...|\n","|(242,[24,69,91,22...|  0.0|       0.0|[0.89002597852031...|\n","|(242,[24,69,91,22...|  0.0|       0.0|[0.89002597852031...|\n","|(242,[24,69,91,22...|  0.0|       0.0|[0.89002597852031...|\n","|(242,[24,69,91,22...|  0.0|       0.0|[0.89002597852031...|\n","|(242,[24,69,91,22...|  0.0|       0.0|[0.89002597852031...|\n","|(242,[24,69,91,22...|  0.0|       0.0|[0.89002597852031...|\n","|(242,[24,69,91,22...|  0.0|       0.0|[0.89002597852031...|\n","|(242,[24,69,91,22...|  0.0|       0.0|[0.89002597852031...|\n","|(242,[24,69,91,22...|  0.0|       0.0|[0.89002597852031...|\n","|(242,[24,69,91,22...|  0.0|       0.0|[0.89002597852031...|\n","|(242,[24,69,91,22...|  0.0|       0.0|[0.89002597852031...|\n","|(242,[24,69,91,22...|  0.0|       0.0|[0.89002597852031...|\n","|(242,[24,69,91,22...|  0.0|       0.0|[0.89002597852031...|\n","+--------------------+-----+----------+--------------------+\n","only showing top 20 rows\n","\n"]}]},{"cell_type":"code","source":["display(pipelineModel.stages[-1], predDF.drop(\"prediction\", \"rawPrediction\", \"probability\"), \"ROC\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":91},"id":"m8Y4e2uZHZNg","executionInfo":{"status":"ok","timestamp":1648920528627,"user_tz":-60,"elapsed":448,"user":{"displayName":"Abdulwahab Abdulliameed","userId":"10732377996348789858"}},"outputId":"3d7c31c0-bcdc-4883-c3fc-e663baead3fc"},"execution_count":48,"outputs":[{"output_type":"display_data","data":{"text/plain":["LogisticRegressionModel: uid=LogisticRegression_a57a643d7e80, numClasses=14, numFeatures=242"]},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["DataFrame[srcip: string, sport: int, dstip: string, dsport: int, proto: string, state: string, dur: double, sbytes: int, dbytes: int, sttl: int, dttl: int, sloss: int, dloss: int, service: string, Sload: double, Dload: double, Spkts: int, Dpkts: int, swin: int, dwin: int, stcpb: bigint, dtcpb: bigint, smeansz: int, dmeansz: int, trans_depth: int, res_bdy_len: int, Sjit: double, Djit: double, Stime: int, Ltime: int, Spkt: double, Dpkt: double, tcprtt: double, synack: double, ackdat: double, is_sm_ips_ports: int, ct_state_ttl: int, ct_flw_http_mthd: int, is_ftp_login: int, ct_ftp_cmd: int, ct_srv_src: int, ct_srv_dst: int, ct_dst_ltm: int, ct_src_ ltm: int, ct_src_dport_ltm: int, ct_dst_sport_ltm: int, ct_dst_src_ltm: int, attack_cat: string, label: double, srcipIndex: double, dstipIndex: double, protoIndex: double, stateIndex: double, srcipOHE: vector, dstipOHE: vector, protoOHE: vector, stateOHE: vector, features: vector]"]},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["'ROC'"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"}},"metadata":{}}]},{"cell_type":"code","source":["#Import MulticlassClassificationEvaluator\n","from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n","#Using the MulticlassClassificationEvaluator to verify the accuracy.\n","mcEvaluator = MulticlassClassificationEvaluator(metricName=\"accuracy\")\n","print(f\"Accuracy:{mcEvaluator.evaluate(predDF)}\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"_TlRQfwrMB6W","executionInfo":{"status":"ok","timestamp":1648918468505,"user_tz":-60,"elapsed":36102,"user":{"displayName":"Abdulwahab Abdulliameed","userId":"10732377996348789858"}},"outputId":"b195513d-1766-4b6f-f3b4-3887fe336bfa"},"execution_count":37,"outputs":[{"output_type":"stream","name":"stdout","text":["Accuracy:0.8729554102762472\n"]}]},{"cell_type":"code","source":["#Import BinaryClassificationEvaluator\n","from pyspark.ml.evaluation import BinaryClassificationEvaluator\n","#Using the BinaryClassificationEvaluator to evaluate the area under the ROC curve\n","bcEvaluator = BinaryClassificationEvaluator(metricName=\"areaUnderROC\")\n","print(f\"Area under ROC curve:{bcEvaluator.evaluate(predDF)}\")"],"metadata":{"id":"Kac2qUh1LwXx"},"execution_count":null,"outputs":[]}],"metadata":{"colab":{"collapsed_sections":[],"name":"Task 3.2 a.) -  Binary classifier over the dataset .ipynb","provenance":[{"file_id":"1k4nfZzfO-xscxEPDGf960ZPzkr7mx59N","timestamp":1648759900501},{"file_id":"19jb2Q7oSOC3zFSm0chXYs4C5LCNpGWT8","timestamp":1648730152751}],"authorship_tag":"ABX9TyN2/7lC70qCjjz8eeY0RydE"},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"}},"nbformat":4,"nbformat_minor":0}