visualize.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"raw_results.tsv\", delimiter='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_dpi = 120"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "legend = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Performance of all attack methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "methods = [\"pgd\", \"tga\", \"degree\", \"random\"]\n",
    "colors = [\"royalblue\", \"orangered\", \"hotpink\", \"dimgray\"]#, \"orangered\"] #, ]\n",
    "shapes = ['-o', '-x', '->', '-+']#,  #, 's']\n",
    "labels = [\"Dyn-PGD\", \"Greedy\", \"Degree\", \"Random\"] #[r\"$\\mathsc{GIN}$\", r\"$\\mathsc{GCN}$\"]#, 'Random']#, 'Degree'] #, 'RL-S2V']\n",
    "\n",
    "sampling = \"td\"\n",
    "epsilons = [0.02, 0.06, 0.10, 0.30, 0.50, 0.70, 0.90]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def quality_plot (df_dm, dataset, model):\n",
    "    fig = plt.figure(figsize=(1.7, 2.0), dpi=my_dpi)\n",
    "    plt.xticks(fontsize = 10)\n",
    "    plt.yticks(fontsize = 10)\n",
    "    for method, col, shape, label in zip(methods, colors, shapes, labels):\n",
    "        df_dmm = df_dm.loc[(df_dm[\"Method\"] == method) & \n",
    "                           (df_dm[\"Sampling\"] == sampling) & \n",
    "                           (df_dm[\"Epsilon\"].isin(epsilons))]\n",
    "        y_mean = df_dmm.groupby(\"Epsilon\").mean()\n",
    "        y_std = df_dmm.groupby(\"Epsilon\").std()\n",
    "        plt.errorbar(epsilons, -y_mean['Perf. Drop %'], yerr=y_std['Perf. Drop %'], \n",
    "                     fmt=shape, color=col, linewidth=1, markersize=6, label=label)\n",
    "    ax = plt.gca()\n",
    "    ax.set_xticks([0.00, 0.25, 0.50, 0.75, 1.0])\n",
    "    if legend:\n",
    "        handles,leglabels = ax.get_legend_handles_labels()\n",
    "        plt.legend(handles, leglabels, loc='best', ncol=1, fontsize=10, frameon=True)\n",
    "    plt.xlabel('Epsilon',fontsize=12)\n",
    "    plt.ylabel('Rel. Drop %',fontsize=12)\n",
    "    fig.tight_layout()\n",
    "    plt.subplots_adjust(top = 1, bottom = 0, right = 1, left = 0, \n",
    "            hspace = 0, wspace = 0)\n",
    "    fig.savefig(f'Plots/{sampling}/results_{dataset}_{model}.pdf', bbox_inches = 'tight', pad_inches = 0.01)\n",
    "    plt.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfg = df.groupby(['Dataset', 'Model'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for dataset, model in dfg.groups.keys():\n",
    "    try:\n",
    "        df_d = dfg.get_group((dataset, model))\n",
    "        quality_plot(df_d, dataset, model)\n",
    "    except:\n",
    "        print (dataset, model)\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "figsize = (1.5*3, 0.2)\n",
    "fig_leg = plt.figure(figsize=figsize)\n",
    "ax_leg = fig_leg.add_subplot(111)\n",
    "# add the legend from the previous axes\n",
    "ax_leg.legend(*ax.get_legend_handles_labels(), ncol=4, loc='center')\n",
    "# hide the axes frame and the x/y labels\n",
    "ax_leg.axis('off')\n",
    "fig_leg.savefig('Plots/legend.pdf', bbox_inches = 'tight', pad_inches = 0.01)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PGD performance across models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "models = [\"gclstm\", \"evgcno\", \"dysat\"]\n",
    "colors = [\"limegreen\", \"mediumorchid\", \"chocolate\"]#, \"orangered\"] #, \"hotpink\"]\n",
    "shapes = ['-v', '-s', '-*']#, '->' #, 's']\n",
    "labels = [\"GCLSTM\", \"EvolveGCN\", \"DySAT\"] #[r\"$\\mathsc{GIN}$\", r\"$\\mathsc{GCN}$\"]#, 'Random']#, 'Degree'] #, 'RL-S2V']\n",
    "\n",
    "sampling = \"rd\"\n",
    "epsilons = [0.02, 0.06, 0.10, 0.30, 0.50, 0.70, 0.90]\n",
    "method = \"pgd\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def quality_plot (df_d, dataset):\n",
    "    fig = plt.figure(figsize=(1.7, 2.0), dpi=my_dpi)\n",
    "    plt.xticks(fontsize = 10)\n",
    "    plt.yticks(fontsize = 10)\n",
    "    for model, col, shape, label in zip(models, colors, shapes, labels):\n",
    "        df_dm = df_d.loc[(df_d[\"Model\"] == model) &\n",
    "                         (df_d[\"Sampling\"] == sampling) &\n",
    "                         (df_d[\"Epsilon\"].isin(epsilons))]\n",
    "        df_dmm = df_dm.loc[df_dm[\"Method\"] == method]\n",
    "        y_mean = df_dmm.groupby(\"Epsilon\").mean()\n",
    "        y_std = df_dmm.groupby(\"Epsilon\").std()\n",
    "        plt.errorbar(epsilons, -y_mean['Perf. Drop %'], yerr=y_std['Perf. Drop %'], \n",
    "                     fmt=shape, color=col, linewidth=1, markersize=6, label=label)\n",
    "        # plt.plot(df_dmm[\"Epsilon\"], df_dmm[\"Perb AUCROC\"], shape, color=col, \n",
    "        #             linewidth=1, markersize=6, label=label)\n",
    "    ax = plt.gca()\n",
    "    if legend:\n",
    "        handles,leglabels = ax.get_legend_handles_labels()\n",
    "        plt.legend(handles, leglabels, loc='best', ncol=1, fontsize=10, frameon=True)\n",
    "    plt.xlabel('Epsilon',fontsize=12)\n",
    "    plt.ylabel(\"Rel. Drop %\",fontsize=12)\n",
    "    fig.tight_layout()\n",
    "    plt.subplots_adjust(top = 1, bottom = 0, right = 1, left = 0, \n",
    "            hspace = 0, wspace = 0)\n",
    "    fig.savefig(f'Plots/{sampling}/results_{dataset}.pdf', bbox_inches = 'tight', pad_inches = 0.01)\n",
    "    plt.close()\n",
    "    return ax\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfg = df.groupby(['Dataset'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for dataset in dfg.groups.keys():\n",
    "    df_d = dfg.get_group(dataset)\n",
    "    ax = quality_plot(df_d, dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "figsize = (1.5*2.0, 0.2)\n",
    "fig_leg = plt.figure(figsize=figsize)\n",
    "ax_leg = fig_leg.add_subplot(111)\n",
    "# add the legend from the previous axes\n",
    "ax_leg.legend(*ax.get_legend_handles_labels(), ncol=4, loc='center')\n",
    "# hide the axes frame and the x/y labels\n",
    "ax_leg.axis('off')\n",
    "fig_leg.savefig('Plots/legend_models.pdf', bbox_inches = 'tight', pad_inches = 0.01)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature perturbation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"raw_results_feat.tsv\", delimiter='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "methods = [\"pgd\", \"random\"]\n",
    "colors = [\"royalblue\", \"dimgray\"]#, \"orangered\"] #, ]\n",
    "shapes = ['-o', '-+']#,  #, 's']\n",
    "labels = [\"Dyn-PGD\", \"Random\"] #[r\"$\\mathsc{GIN}$\", r\"$\\mathsc{GCN}$\"]#, 'Random']#, 'Degree'] #, 'RL-S2V']\n",
    "\n",
    "sampling = \"rd\"\n",
    "epsilons = [0.02, 0.06, 0.10, 0.30, 0.50, 0.70, 0.90]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def quality_plot (df_dm, dataset, model):\n",
    "    fig = plt.figure(figsize=(1.7, 2.0), dpi=my_dpi)\n",
    "    plt.xticks(fontsize = 10)\n",
    "    plt.yticks(fontsize = 10)\n",
    "    for method, col, shape, label in zip(methods, colors, shapes, labels):\n",
    "        df_dmm = df_dm.loc[(df_dm[\"Method\"] == method) & \n",
    "                           (df_dm[\"Sampling\"] == sampling) & \n",
    "                           (df_dm[\"Epsilon\"].isin(epsilons))]\n",
    "        y_mean = df_dmm.groupby(\"Epsilon\").mean()\n",
    "        y_std = df_dmm.groupby(\"Epsilon\").std()\n",
    "        plt.errorbar(epsilons, -y_mean['Perf. Drop %'], yerr=y_std['Perf. Drop %'], \n",
    "                     fmt=shape, color=col, linewidth=1, markersize=6, label=label)\n",
    "    ax = plt.gca()\n",
    "    ax.set_xticks([0.00, 0.25, 0.50, 0.75, 1.0])\n",
    "    if legend:\n",
    "        handles,leglabels = ax.get_legend_handles_labels()\n",
    "        plt.legend(handles, leglabels, loc='best', ncol=1, fontsize=10, frameon=True)\n",
    "    plt.xlabel('Epsilon',fontsize=12)\n",
    "    plt.ylabel('Rel. Drop %',fontsize=12)\n",
    "    fig.tight_layout()\n",
    "    plt.subplots_adjust(top = 1, bottom = 0, right = 1, left = 0, \n",
    "            hspace = 0, wspace = 0)\n",
    "    fig.savefig(f'Plots/feat/{sampling}/results_{dataset}_{model}.pdf', bbox_inches = 'tight', pad_inches = 0.01)\n",
    "    plt.close()\n",
    "    return ax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfg = df.groupby(['Dataset', 'Model'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for dataset, model in dfg.groups.keys():\n",
    "    df_d = dfg.get_group((dataset, model))\n",
    "    ax = quality_plot(df_d, dataset, model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "figsize = (1.5*2, 0.2)\n",
    "fig_leg = plt.figure(figsize=figsize)\n",
    "ax_leg = fig_leg.add_subplot(111)\n",
    "# add the legend from the previous axes\n",
    "ax_leg.legend(*ax.get_legend_handles_labels(), ncol=4, loc='center')\n",
    "# hide the axes frame and the x/y labels\n",
    "ax_leg.axis('off')\n",
    "fig_leg.savefig('Plots/legend_feat.pdf', bbox_inches = 'tight', pad_inches = 0.01)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Online perturbation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"raw_results_online3.tsv\", delimiter='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "methods = [\"pgd\", \"degree\", \"random\"]\n",
    "colors = [\"royalblue\", \"hotpink\", \"dimgray\"]#, \"orangered\"] #, ]\n",
    "shapes = ['-o', '->', '-+']#,  #, 's']\n",
    "labels = [\"Dyn-PGD\", \"Degree\", \"Random\"] #[r\"$\\mathsc{GIN}$\", r\"$\\mathsc{GCN}$\"]#, 'Random']#, 'Degree'] #, 'RL-S2V']\n",
    "\n",
    "sampling = \"rd\"\n",
    "epsilons = [0.02, 0.06, 0.10, 0.30, 0.50, 0.70, 0.90]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def quality_plot (df_dm, dataset, model):\n",
    "    fig = plt.figure(figsize=(1.7, 2.0), dpi=my_dpi)\n",
    "    plt.xticks(fontsize = 10)\n",
    "    plt.yticks(fontsize = 10)\n",
    "    for method, col, shape, label in zip(methods, colors, shapes, labels):\n",
    "        df_dmm = df_dm.loc[(df_dm[\"Method\"] == method) & \n",
    "                           (df_dm[\"Sampling\"] == sampling) & \n",
    "                           (df_dm[\"Epsilon\"].isin(epsilons))]\n",
    "        y_mean = df_dmm.groupby(\"Epsilon\").mean()\n",
    "        y_std = df_dmm.groupby(\"Epsilon\").std()\n",
    "        plt.errorbar(epsilons, -y_mean['Perf. Drop %'], yerr=y_std['Perf. Drop %'], \n",
    "                     fmt=shape, color=col, linewidth=1, markersize=6, label=label)\n",
    "    ax = plt.gca()\n",
    "    ax.set_xticks([0.00, 0.25, 0.50, 0.75, 1.0])\n",
    "    if legend:\n",
    "        handles,leglabels = ax.get_legend_handles_labels()\n",
    "        plt.legend(handles, leglabels, loc='best', ncol=1, fontsize=10, frameon=True)\n",
    "    plt.xlabel('Epsilon',fontsize=12)\n",
    "    plt.ylabel('Rel. Drop %',fontsize=12)\n",
    "    fig.tight_layout()\n",
    "    plt.subplots_adjust(top = 1, bottom = 0, right = 1, left = 0, \n",
    "            hspace = 0, wspace = 0)\n",
    "    fig.savefig(f'Plots/online/{sampling}/results_{dataset}_{model}.pdf', bbox_inches = 'tight', pad_inches = 0.01)\n",
    "    plt.close()\n",
    "    return ax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfg = df.groupby(['Dataset', 'Model'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for dataset, model in dfg.groups.keys():\n",
    "    try:\n",
    "        df_dm = dfg.get_group((dataset, model))\n",
    "        ax = quality_plot(df_dm, dataset, model)\n",
    "    except:\n",
    "        print (dataset, model)\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "figsize = (1.5*2, 0.2)\n",
    "fig_leg = plt.figure(figsize=figsize)\n",
    "ax_leg = fig_leg.add_subplot(111)\n",
    "# add the legend from the previous axes\n",
    "ax_leg.legend(*ax.get_legend_handles_labels(), ncol=4, loc='center')\n",
    "# hide the axes frame and the x/y labels\n",
    "ax_leg.axis('off')\n",
    "fig_leg.savefig('Plots/legend_online.pdf', bbox_inches = 'tight', pad_inches = 0.01)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Target-specific metric plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "methods = [\"pgd\", \"tga\", \"degree\", \"random\"]\n",
    "colors = [\"royalblue\", \"orangered\", \"hotpink\", \"dimgray\"]#, \"orangered\"] #, ]\n",
    "shapes = ['-o', '-x', '->', '-+']#,  #, 's']\n",
    "labels = [\"Dyn-PGD\", \"Greedy\", \"Degree\", \"Random\"] #[r\"$\\mathsc{GIN}$\", r\"$\\mathsc{GCN}$\"]#, 'Random']#, 'Degree'] #, 'RL-S2V']\n",
    "\n",
    "sampling = \"rd\"\n",
    "epsilons = [0.02, 0.06, 0.10, 0.30, 0.50, 0.70, 0.90]\n",
    "metric = \"E\"\n",
    "cohen_d = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def quality_plot (df_dm, dataset, model):\n",
    "    fig = plt.figure(figsize=(1.7, 2.0), dpi=my_dpi)\n",
    "    plt.xticks(fontsize = 10)\n",
    "    plt.yticks(fontsize = 10)\n",
    "    for method, col, shape, label in zip(methods, colors, shapes, labels):\n",
    "        df_dm = df_dm.dropna(subset=[metric])\n",
    "        df_dm[f'{metric}_mean'] = [float(x.split(\", \")[0]) for x in df_dm[metric]]\n",
    "        df_dm[f'{metric}_std'] = [float(x.split(\", \")[1]) for x in df_dm[metric]]\n",
    "        df_dmm = df_dm.loc[(df_dm[\"Method\"] == method) & \n",
    "                        (df_dm[\"Sampling\"] == sampling) & \n",
    "                        (df_dm[\"Epsilon\"].isin(epsilons))]\n",
    "        y_mean = df_dmm.groupby(\"Epsilon\").mean()\n",
    "        if (cohen_d):\n",
    "            y_mean[f'{metric}_cohen'] = (y_mean[f'{metric}_mean'] - 0)/y_mean[f'{metric}_std']\n",
    "            plt.plot(epsilons, y_mean[f'{metric}_cohen'], shape, color=col, linewidth=1, markersize=6, label=label)\n",
    "        else:\n",
    "            plt.errorbar(epsilons, y_mean[f'{metric}_mean'], yerr=y_mean[f'{metric}_std'], \n",
    "                        fmt=shape, color=col, linewidth=1, markersize=6, label=label)\n",
    "    ax = plt.gca()\n",
    "    ax.set_xticks([0.00, 0.25, 0.50, 0.75, 1.0])\n",
    "    if legend:\n",
    "        handles,leglabels = ax.get_legend_handles_labels()\n",
    "        plt.legend(handles, leglabels, loc='best', ncol=1, fontsize=10, frameon=True)\n",
    "    plt.xlabel('Epsilon',fontsize=12)\n",
    "    if cohen_d:\n",
    "        plt.ylabel(f'{metric}_cohen',fontsize=12)\n",
    "    else:\n",
    "        plt.ylabel(f'{metric}',fontsize=12)\n",
    "    fig.tight_layout()\n",
    "    plt.subplots_adjust(top = 1, bottom = 0, right = 1, left = 0, hspace = 0, wspace = 0)\n",
    "    if cohen_d:\n",
    "        fig.savefig(f'Plots/{metric}_cohen/{sampling}/results_{dataset}_{model}.pdf', bbox_inches = 'tight', pad_inches = 0.01)\n",
    "    else:\n",
    "        fig.savefig(f'Plots/{metric}/{sampling}/results_{dataset}_{model}.pdf', bbox_inches = 'tight', pad_inches = 0.01)\n",
    "    plt.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfg = df.groupby(['Dataset', 'Model'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for dataset, model in dfg.groups.keys():\n",
    "    # try:\n",
    "    df_d = dfg.get_group((dataset, model))\n",
    "    quality_plot(df_d, dataset, model)\n",
    "    # except:\n",
    "    #     print (dataset, model)\n",
    "    #     pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# E metric statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "methods = [\"pgd\", \"tga\", \"degree\", \"random\"]\n",
    "\n",
    "sampling = \"rd\"\n",
    "epsilons = [0.02, 0.06, 0.10, 0.30, 0.50, 0.70, 0.90]\n",
    "metric = \"E\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def resdata_append (df_dm, dataset, model, res_df):\n",
    "    for method in methods:\n",
    "        df_dm = df_dm.dropna(subset=[metric])\n",
    "        df_dm[f'{metric}_mean'] = [float(x.split(\", \")[0]) for x in df_dm[metric]]\n",
    "        df_dm[f'{metric}_std'] = [float(x.split(\", \")[1]) for x in df_dm[metric]]\n",
    "        df_dmm = df_dm.loc[(df_dm[\"Method\"] == method) & \n",
    "                        (df_dm[\"Sampling\"] == sampling) & \n",
    "                        (df_dm[\"Epsilon\"].isin(epsilons))]\n",
    "        y_mean = df_dmm.groupby(\"Epsilon\").mean()\n",
    "        y_std = df_dmm.groupby(\"Epsilon\").std()\n",
    "        cohen_d = (y_mean[f'{metric}_mean'] - 0)/y_mean[f'{metric}_std']\n",
    "        for epsilon in cohen_d.index:\n",
    "            res_df[\"Dataset\"].append(dataset)\n",
    "            res_df[\"Model\"].append(model)\n",
    "            res_df[\"Method\"].append(method)\n",
    "            res_df[\"Epsilon\"].append(epsilon)\n",
    "            res_df[\"Mean Perf. Drop %\"].append(y_mean['Perf. Drop %'].loc[epsilon])\n",
    "            res_df[\"Std Perf. Drop %\"].append(y_std['Perf. Drop %'].loc[epsilon])\n",
    "            res_df[\"Cohen_d\"].append(cohen_d.loc[epsilon])\n",
    "            res_df[f'{metric}_mean'].append(y_mean[f'{metric}_mean'].loc[epsilon])\n",
    "            res_df[f'{metric}_std'].append(y_mean[f'{metric}_std'].loc[epsilon])\n",
    "    return res_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfg = df.groupby(['Dataset', 'Model'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "\n",
    "res_df = defaultdict(lambda: [])\n",
    "\n",
    "for dataset, model in dfg.groups.keys():\n",
    "    df_d = dfg.get_group((dataset, model))\n",
    "    res_df = resdata_append(df_d, dataset, model, res_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res_df = pd.DataFrame(res_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res_df_tab = res_df.loc[(res_df['Epsilon'] == 0.5) & ~(res_df['Dataset'].isin([\"DBLP\"])) & (res_df[\"Method\"] == 'pgd')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res_df_tab[['Dataset', 'Model', 'Mean Perf. Drop %',\n",
    "       'Std Perf. Drop %', 'Cohen_d', 'E_mean', 'E_std']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res_df.to_csv(\"e.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Running Time experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_reddit_mean = [4.01380466, 4.248379645, 3.971783389, 5.688414611, 6.588495526, 6.306535423, 3.106602286]\n",
    "random_reddit_std = [1.506646785, 1.344604474, 1.37294442, 2.953934616, 4.031191687, 6.299767277, 2.720999723]\n",
    "\n",
    "degree_reddit_mean = [4.274370613, 4.57556869, 4.00092563, 4.066166203, 4.419590709, 4.114394008, 4.065688193]\n",
    "degree_reddit_std = [1.230938658, 1.337842071, 1.442863082, 1.167981418, 1.414808568, 1.258317752, 1.278922633]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "methods = [\"pgd\", \"tga\", \"degree\", \"random\"]\n",
    "colors = [\"royalblue\", \"orangered\", \"hotpink\", \"dimgray\"]#, \"orangered\"] #, ]\n",
    "shapes = ['-o', '-x', '->', '-+']#,  #, 's']\n",
    "labels = [\"Dyn-PGD\", \"Greedy\", \"Degree\", \"Random\"] #[r\"$\\mathsc{GIN}$\", r\"$\\mathsc{GCN}$\"]#, 'Random']#, 'Degree'] #, 'RL-S2V']\n",
    "\n",
    "sampling = \"rd\"\n",
    "epsilons = [0.02, 0.06, 0.10, 0.30, 0.50, 0.70, 0.90]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def quality_plot (df_dm, dataset, model):\n",
    "    fig = plt.figure(figsize=(1.7, 2.0), dpi=my_dpi)\n",
    "    plt.xticks(fontsize = 10)\n",
    "    plt.yticks(fontsize = 10)\n",
    "    for method, col, shape, label in zip(methods, colors, shapes, labels):\n",
    "        if method == \"random\" and dataset == \"reddit_hyperlinks\":\n",
    "            plt.errorbar(epsilons, random_reddit_mean, yerr=random_reddit_std, \n",
    "                     fmt=shape, color=col, linewidth=1, markersize=6, label=label)\n",
    "        elif method == \"degree\" and dataset == \"reddit_hyperlinks\":\n",
    "            plt.errorbar(epsilons, degree_reddit_mean, yerr=degree_reddit_std, \n",
    "                     fmt=shape, color=col, linewidth=1, markersize=6, label=label)\n",
    "        else:\n",
    "            df_dmm = df_dm.loc[(df_dm[\"Method\"] == method) & \n",
    "                            (df_dm[\"Sampling\"] == sampling) & \n",
    "                            (df_dm[\"Epsilon\"].isin(epsilons))]\n",
    "            df_dmm['Time taken'] = [float(x) for x in df_dmm[\"Time taken\"]]\n",
    "            y_mean = df_dmm.groupby(\"Epsilon\").mean()\n",
    "            y_std = df_dmm.groupby(\"Epsilon\").std()\n",
    "            # print (method, model, dataset, y_mean.columns, y_std.columns)\n",
    "            plt.errorbar(epsilons, y_mean['Time taken'], yerr=y_std['Time taken'], \n",
    "                        fmt=shape, color=col, linewidth=1, markersize=6, label=label)\n",
    "    ax = plt.gca()\n",
    "    ax.set_xticks([0.00, 0.25, 0.50, 0.75, 1.0])\n",
    "    if legend:\n",
    "        handles,leglabels = ax.get_legend_handles_labels()\n",
    "        plt.legend(handles, leglabels, loc='best', ncol=1, fontsize=10, frameon=True)\n",
    "    plt.xlabel('Epsilon',fontsize=12)\n",
    "    plt.ylabel('Time/target (s)',fontsize=12)\n",
    "    fig.tight_layout()\n",
    "    plt.subplots_adjust(top = 1, bottom = 0, right = 1, left = 0, \n",
    "            hspace = 0, wspace = 0)\n",
    "    fig.savefig(f'Plots/Time/{sampling}/results_{dataset}_{model}.pdf', bbox_inches = 'tight', pad_inches = 0.01)\n",
    "    plt.close()\n",
    "    return ax\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfg = df.groupby(['Dataset', 'Model'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for dataset, model in dfg.groups.keys():\n",
    "    try:\n",
    "        df_d = dfg.get_group((dataset, model))\n",
    "        quality_plot(df_d, dataset, model)\n",
    "    except:\n",
    "        print (dataset, model)\n",
    "        pass"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "e75189700533a6f9c77aab1809c5976db3a8fb6574f86c0d637a13ec3af6e296"
  },
  "kernelspec": {
   "display_name": "Python 3.9.5 64-bit ('base': conda)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}