vamishing_grad + log grad.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09266f2c-5002-49ef-aa5e-93259cda7b25",
   "metadata": {},
   "outputs": [],
   "source": [
    "!echo $http_proxy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "368e9b62-c882-4022-ba0a-974b2b5cdea0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import comet_ml\n",
    "comet_ml.login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a456f5a-08e0-4585-94a7-6d28aaf05af7",
   "metadata": {},
   "outputs": [],
   "source": [
    "from comet_ml import Experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "05b17100-1d4d-4c51-b058-ce12d77b642a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras\n",
    "import numpy as np\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout\n",
    "from keras.callbacks import EarlyStopping\n",
    "from keras.preprocessing import image\n",
    "from keras.optimizers import RMSprop, Adam,SGD\n",
    "\n",
    "\n",
    "from keras.utils import to_categorical\n",
    "from tensorflow.keras.losses import categorical_crossentropy\n",
    "from keras.callbacks import Callback\n",
    "import logging\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ec787f8-e5c6-46d7-acca-9844fc6a31b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "x_train = x_train.reshape(-1, 28, 28, 1).astype('float32') / 255\n",
    "x_test = x_test.reshape(-1, 28, 28, 1).astype('float32') / 255\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "665bfce6-13ff-4c27-bad1-fd1294f5f03a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert class vectors to binary class matrices\n",
    "y_train = keras.utils.to_categorical(y_train, 10)\n",
    "y_test = keras.utils.to_categorical(y_test, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0de6572c-5ea8-467b-827b-4ead699e218d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_vanish_grad(input_shape=(28, 28, 1), numnodes=128):\n",
    "    model = Sequential()\n",
    "\n",
    "    # Input layer (Flatten the 28x28 image to 784)\n",
    "    model.add(Flatten(input_shape=input_shape))  # Flatten layer to reshape (28,28,1) to (784,)\n",
    "\n",
    "    # First hidden layer\n",
    "    model.add(Dense(numnodes, activation='sigmoid'))\n",
    "\n",
    "    # Hidden layers: Increasing depth to induce vanishing gradients\n",
    "    for _ in range(10):  # Add 10 hidden layers\n",
    "        model.add(Dense(numnodes, activation='sigmoid'))\n",
    "\n",
    "    # Output layer\n",
    "    model.add(Dense(10, activation='softmax'))  # Output for 10-class classification\n",
    "\n",
    "    return model\n",
    "\n",
    "# Ensure the correct input shape (28, 28, 1) for the image\n",
    "model = build_vanish_grad(numnodes=128, input_shape=(28, 28, 1))\n",
    "# Compile with high learning rate and no gradient control\n",
    "model.compile(optimizer=SGD(learning_rate=0.00001), loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "model.summary()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0cf8a06d-7d5b-4f80-9f9b-ecb156782e9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment = Experiment(project_name=\"Vanishing_grad\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9657f791-e2af-41a2-8157-b1fdb9dee38a",
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment.display()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ec1a66a-1248-46b1-af2f-97d153cd2dce",
   "metadata": {},
   "outputs": [],
   "source": [
    "class LoggingCallback(Callback):\n",
    "    def __init__(self, experiment):\n",
    "        self.experiment = experiment\n",
    "\n",
    "    def on_epoch_end(self, epoch, logs=None):\n",
    "        count = 0\n",
    "        for weight in self.model.get_weights():\n",
    "            name = \"layer-\" + str((count // 2) + 1) + \"-\" + (\"bias\" if count % 2 == 1 else \"weight\")\n",
    "            self.experiment.log_histogram_3d(weight, name=name)\n",
    "            count += 1\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "725aa40d-4e71-401e-a8f0-0e1c5d1e212c",
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_fn = tf.keras.losses.CategoricalCrossentropy()  # Adjust as per your task\n",
    "\n",
    "# Training loop\n",
    "\n",
    "for epoch in range(30):\n",
    "    with tf.GradientTape() as tape:\n",
    "        predictions = model(x_test)\n",
    "        loss = loss_fn(y_test, predictions)\n",
    "    grads = tape.gradient(loss, model.trainable_variables)\n",
    "\n",
    "            # Log gradients as histograms\n",
    "    for i, grad in enumerate(grads):\n",
    "        experiment.log_histogram_3d(\n",
    "            grad.numpy().flatten(), name=f\"Gradient_Layer_{i}\", step=epoch\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "55945335-c183-4611-ad17-da6bc93ee89c",
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment.log_dataset_hash(x_train)\n",
    "\n",
    "cb = LoggingCallback(experiment)\n",
    "with experiment.train():\n",
    "    model.fit(\n",
    "        x_train,\n",
    "        y_train,\n",
    "        batch_size=32,\n",
    "        epochs=30,\n",
    "        callbacks=[cb],\n",
    "        validation_data=(x_test, y_test)\n",
    "    )\n",
    "\n",
    "# callbacks=[cb],\n",
    "score = model.evaluate(x_test, y_test, verbose=0)\n",
    "logging.info(\"Score %s\", score)\n",
    "experiment.end()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "193264c0-e8bb-4741-9373-1f5a41aa6e61",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c5d6f5ff-e415-408a-bd76-6ffbf98f8a37",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bdc19acf-1ce4-4fe2-8f33-afbe61db3867",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f21a3d8-acff-4efe-aef5-7ab7766b9442",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}