run_training.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "run_training.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyOTfz/ct3JhoCo37G/apCiC",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jfgd/hornet-detector-trainer/blob/gcolab/run_training.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LZI6oVdvVZBO",
        "colab_type": "text"
      },
      "source": [
        "Run Tensorflow Model Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ex2LAq7U6ADL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Somehow needed ...\n",
        "!pip3 install -U numpy==1.17.4"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oZt8zh9EyiVG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Stay to tensorflow 1.x for now\n",
        "%tensorflow_version 1.x"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZoL6Nz-EgmjB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Mount google drive to save training checkpoints and results\n",
        "from google.colab import drive\n",
        "drive.mount('/gdrive')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9vT22zl7exCk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Clone the entire repo\n",
        "!rm -rf hornet-detector-trainer\n",
        "!git clone --depth 1 https://github.com/jfgd/hornet-detector-trainer.git\n",
        "%cd hornet-detector-trainer\n",
        "!ls"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nWJhmbGFcyxR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Print some infos\n",
        "!git log\n",
        "!du -sh\n",
        "!du -sh .git\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PyWCnnCmgo4i",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Create needed folder hierarchy in gdrive\n",
        "!mkdir -p /gdrive/My\\ Drive/hornet-detector-trainer/training\n",
        "!mkdir -p /gdrive/My\\ Drive/hornet-detector-trainer/graphs\n",
        "!ls -l /gdrive/My\\ Drive/hornet-detector-trainer/\n",
        "!ls -l /gdrive/My\\ Drive/hornet-detector-trainer/*"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TbYvwzhKgmhy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Prepare folders to use gdrive, useful to recover from a checkpoint\n",
        "!pwd\n",
        "!mv training training.fromgit\n",
        "!ln -s /gdrive/My\\ Drive/hornet-detector-trainer/training training\n",
        "!cp training.fromgit/* training/\n",
        "!rm -rf graphs\n",
        "!ln -s /gdrive/My\\ Drive/hornet-detector-trainer/graphs graphs\n",
        "!ls -l\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "62-oSf3eexBT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Pull tensorflow models\n",
        "!make models"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "32KaPP8mPwy4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Generate CSV files\n",
        "!make csv"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QJg6MfS5P0e5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Some stats\n",
        "!./scripts/check_img.sh images | tee training/nb_images.txt"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t6QgjU54qwlS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Generate record files\n",
        "!make record"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aCn-IfE318oB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Run tensorboard\n",
        "!wget https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip\n",
        "!unzip -o ngrok-stable-linux-amd64.zip\n",
        "LOG_DIR = \"training\"\n",
        "get_ipython().system_raw(\n",
        "    'tensorboard --samples_per_plugin images=50 --logdir {} --host 0.0.0.0 --port 6006 &'\n",
        "    .format(LOG_DIR)\n",
        ")\n",
        "get_ipython().system_raw('./ngrok http 6006 &')\n",
        "! curl -s http://localhost:4040/api/tunnels | python3 -c \\\n",
        "    \"import sys, json; print(json.load(sys.stdin)['tunnels'][0]['public_url'])\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l5LX4X7p0wp1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Train\n",
        "!make train\n",
        "!ls training"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rm_9wuFAIF21",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Print TensorFlow version\n",
        "import tensorflow as tf\n",
        "print(\"Tensorflow version: \", tf.__version__)\n",
        "tfver = str(tf.__version__)\n",
        "!touch graphs/exported_with_tf_{tfver}\n",
        "!ls graphs"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bWZl_Vvq50Q4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Export Graph\n",
        "!cp training/nb_images.txt \n",
        "!make export-graph\n",
        "!ls graphs"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}