scripts/Ground_truth_export_and_thumbnails.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "0",
   "metadata": {},
   "source": [
    "# Ground truth counting and exporting\n",
    "This script is what we used to validate our cell counts done in whole or in part in other notebooks (those usually focus just on test data, not training). "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "from pathlib import Path\n",
    "from cellpose_omni import plot, models, core, utils, io, dynamics\n",
    "import skimage.io\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "plt.style.use('dark_background')\n",
    "import matplotlib as mpl\n",
    "%matplotlib inline\n",
    "mpl.rcParams['figure.dpi'] = 300\n",
    "import time, os, sys\n",
    "from tifffile import imread\n",
    "# use_GPU = core.use_gpu()\n",
    "# print('>>> GPU activated? %d'%use_GPU)\n",
    "\n",
    "import omnipose"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2",
   "metadata": {},
   "outputs": [],
   "source": [
    "base = '/home/kcutler/DataDrive/omnipose_all'\n",
    "dataset = ['phase','fluor']\n",
    "subset = ['train','test']\n",
    "basedir = [[os.path.join(base,d,s + '_sorted') for s in subset] for d in dataset]\n",
    "\n",
    "base = '/home/kcutler/DataDrive/omnipose_all/OSF/'\n",
    "dataset = ['worm']\n",
    "subset = ['train','test']\n",
    "basedir = [[os.path.join(base,d,s) for s in subset] for d in dataset]\n",
    "\n",
    "# base = '/home/kcutler/DataDrive/'\n",
    "# dataset = ['luca']\n",
    "# subset = ['train','test']\n",
    "# basedir = [[os.path.join(base,d,s) for s in subset] for d in dataset]\n",
    "\n",
    "def getname(path,suffix='_masks'):\n",
    "    return os.path.splitext(Path(path).name)[0].replace(suffix,'')\n",
    "\n",
    "def get_GT(basedir):\n",
    "    mask_filter = '_masks'\n",
    "    img_names = io.get_image_files(basedir,mask_filter,look_one_level_down=True)\n",
    "    mask_names = io.get_label_files(img_names, mask_filter)\n",
    "    #annoyingly, the names are not sorted properly because the paths are all different for the test folder...\n",
    "    mask_names = sorted(mask_names,key=getname)\n",
    "    img_names = sorted(img_names,key=getname)\n",
    "    imgs = [skimage.io.imread(f) for f in img_names]\n",
    "    # masks_gt = [utils.format_labels(utils.clean_boundary(skimage.io.imread(f))) for f in mask_names]\n",
    "    masks_gt = [omnipose.utils.format_labels(omnipose.utils.clean_boundary(skimage.io.imread(f)),clean=True) for f in mask_names]\n",
    "    return imgs, masks_gt, img_names, mask_names\n",
    "\n",
    "J = len(dataset)\n",
    "K = len(subset)\n",
    "imgs = [[[] for k in range(K)] for j in range(J)]\n",
    "masks_gt = [[[] for k in range(K)] for j in range(J)]\n",
    "img_names = [[[] for k in range(K)] for j in range(J)]\n",
    "mask_names = [[[] for k in range(K)] for j in range(J)]\n",
    "for j in range(J):\n",
    "    for k in range(K):\n",
    "        print((basedir[j][k]))\n",
    "        imgs[j][k], masks_gt[j][k], img_names[j][k], mask_names[j][k] = get_GT(basedir[j][k])\n",
    "# train_imgs, train_masks, train_img_names, train_mask_names = get_GT(train_dir)\n",
    "# test_imgs, test_masks, test_img_names, train_mask_names = get_GT(train_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3",
   "metadata": {},
   "source": [
    "### Count images and cells in dataset\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Count images\n",
    "import fastremap, ncolor\n",
    "from matplotlib.colors import ListedColormap\n",
    "from omnipose import utils\n",
    "from cellpose_omni import plot\n",
    "cmap = ListedColormap([color for color in utils.sinebow(5).values()])\n",
    "\n",
    "imagecount = np.zeros([J,K],int)\n",
    "cellcount = np.zeros([J,K],int)\n",
    "show = False\n",
    "for j in range(J):\n",
    "    for k in range(K):\n",
    "        c = len(imgs[j][k])\n",
    "        imagecount[j,k] += c\n",
    "\n",
    "        for maski,imgi,name in zip(masks_gt[j][k],imgs[j][k],img_names[j][k]):\n",
    "            # print(name)\n",
    "            c =  len(fastremap.unique(maski))\n",
    "            cellcount[j,k] += c\n",
    "            if maski.shape!=imgi.shape:\n",
    "                print(name,'has mask mismatch')\n",
    "            if show:\n",
    "                fig = plt.figure(figsize=(10,10))\n",
    "                p = utils.normalize99(imgi)\n",
    "                img0 = np.stack((p,p,p),axis=2)\n",
    "                outli = plot.outline_view(img0,maski)\n",
    "\n",
    "                pic = np.hstack((img0,utils.rescale(outli),cmap(ncolor.label(maski))[:,:,:3]))\n",
    "                plt.imshow(pic)\n",
    "                plt.axis('off')\n",
    "                plt.show()\n",
    "\n",
    "    print(np.sum(imagecount[j]),np.sum(cellcount[j]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5",
   "metadata": {},
   "outputs": [],
   "source": [
    "basedir = '/home/kcutler/DataDrive/omnipose_all/'\n",
    "\n",
    "\n",
    "\n",
    "for j in range(J):\n",
    "    text = []\n",
    "    text += ['{} dataset\\n'.format(dataset[j])]\n",
    "    for k in range(K):\n",
    "        text+=[subset[k]+':',\n",
    "               '\\t image count: {}'.format(imagecount[j,k]),\n",
    "               '\\t cell count: {}'.format(cellcount[j,k])]\n",
    "    text+=['total:',\n",
    "           '\\t image count: {}'.format(np.sum(imagecount[j])),\n",
    "           '\\t cell count: {}'.format(np.sum(cellcount[j])),'\\n']\n",
    "    \n",
    "    with open(os.path.join(basedir,'{}_datatet_stats.txt'.format(dataset[j])), \"w\") as text_file:\n",
    "        [print(t,file=f) for t in text for f in [None,text_file]]\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6",
   "metadata": {},
   "outputs": [],
   "source": [
    "J"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7",
   "metadata": {},
   "source": [
    "## Check to make sure that there is no duplication between test and train datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8",
   "metadata": {},
   "outputs": [],
   "source": [
    "for j in range(J):\n",
    "    # A = ['a','b','c']\n",
    "    # B = ['c']\n",
    "    A = img_names[j][0] #assuming two categories \n",
    "    B = img_names[j][1]\n",
    "    print([x for x in A if x in B])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9",
   "metadata": {},
   "source": [
    "### Export images, masks, and thumbnails"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10",
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "import tifffile\n",
    "from omnipose.utils import rescale, format_labels\n",
    "offset = [0,len(imgs[0])]\n",
    "# cmap2 = mpl.cm.get_cmap('gray')\n",
    "# cmap2 = mpl.cm.get_cmap('magma')\n",
    "\n",
    "clean = 1\n",
    "save_ncolor = 0\n",
    "save_color = 0\n",
    "display = 0\n",
    "mrcnn = 0 # export to mrcnn folder\n",
    "matlab = 0 # export for morphometrics and supersegger processing\n",
    "\n",
    "# get all the jpg files from the current folder\n",
    "for j in range(J):\n",
    "# for j in [1]:\n",
    "    for k in range(K):\n",
    "        # for f,im in zip(img_names[k],imgs[k]):\n",
    "        for i,im,mask in zip(range(len(imgs[j][k])),imgs[j][k],masks_gt[j][k]):\n",
    "            # name = 'img'+str(i+offset[k])\n",
    "            # name = '%03d' % (i+offset[k],) + '_img'\n",
    "            name = '%03d' % (i,) \n",
    "            if mask.shape!=im.shape:\n",
    "                print(name,'has mask mismatch')\n",
    "            if clean:\n",
    "                #save a copy with new naming convention        \n",
    "                savedir = basedir[j][k]+'_website'\n",
    "                io.check_dir(savedir)\n",
    "                io.imsave(os.path.join(savedir,name+'_img.tif'),im)  \n",
    "                io.imsave(os.path.join(savedir,name+'_masks.tif'),mask)    \n",
    "\n",
    "                # convert to thumbnail image\n",
    "                im = (omnipose.utils.normalize99(im)*255).astype(np.uint8)\n",
    "                im = Image.fromarray(im)\n",
    "                im.thumbnail((128, 128), Image.ANTIALIAS)\n",
    "                savedir = basedir[j][k]+'_website_thumbnails'\n",
    "                io.check_dir(savedir)\n",
    "                # im.save(os.path.join(savedir,getname(f)+'_thumbnail.tif'))\n",
    "                im.save(os.path.join(savedir,name+'_thumbnail.tif'))\n",
    "\n",
    "            #Also save an ncolor version\n",
    "            if save_ncolor:\n",
    "                pic = cmap(ncolor.label(mask))\n",
    "                # pic[:,:,-1] = mask>0 #transparency, not needed\n",
    "                savedir = basedir[j][k]+'_ncolor'\n",
    "                io.check_dir(savedir)\n",
    "                skimage.io.imsave(os.path.join(savedir,name+'_ncolor_masks.tif'),(pic*(2**8-1)).astype(np.uint8))\n",
    "\n",
    "            # can also save rescaled version of label matrix\n",
    "            if save_color:\n",
    "                cmap2 = ListedColormap([color for color in utils.sinebow(mask.max()).values()])\n",
    "                pic = cmap2(omnipose.utils.rescale(mask))\n",
    "                pic[:,:,-1] = mask>0 #transparency\n",
    "                savedir = basedir[j][k]+'_rescaled_sinebow'\n",
    "                io.check_dir(savedir)\n",
    "                skimage.io.imsave(os.path.join(savedir,name+'_masks_rescaled.tif'),(pic*(2**8-1)).astype(np.uint8))\n",
    "\n",
    "            if display:\n",
    "                fig = plt.figure(figsize=[4]*2)\n",
    "                plt.imshow(np.hstack((utils.rescale(im),ncolor.label(mask))))\n",
    "                plt.axis('off')\n",
    "                plt.show()\n",
    "\n",
    "            if j==0: # only meant for phase \n",
    "                if mrcnn:\n",
    "                    name = getname(mask_names[j][k][i]) # use original names for this export \n",
    "                    savedir = os.path.join('/home/kcutler/DataDrive/omnipose_all/RGB_for_mrcnn',subset[k])\n",
    "                    io.check_dir(savedir)\n",
    "                    # io.imsave(os.path.join(savedir,name+'_img.tif'),np.stack([im]*3))  \n",
    "                    # io.imsave(os.path.join(savedir,name+'_masks.tif'),mask)    \n",
    "\n",
    "                    mgt = omnipose.utils.format_labels(mask,clean=True)\n",
    "                    tifffile.imsave(os.path.join(savedir,name+'_masks.tif'),mgt.astype('int64'))\n",
    "\n",
    "                    img = np.repeat((rescale(im)*(2**16-1)).astype('uint16')[:,:,np.newaxis],repeats=3,axis=-1) # must use rgb images\n",
    "                    tifffile.imsave(os.path.join(savedir,name+'.tif'),img) \n",
    "\n",
    "\n",
    "                if matlab and subset[k]=='test':\n",
    "                    name = getname(mask_names[j][k][i]) # use original names for this export \n",
    "                    savedir = os.path.join('/home/kcutler/DataDrive/omnipose_all/matlab')\n",
    "                    io.check_dir(savedir)     \n",
    "                    img = (rescale(im)*(2**16-1)).astype('uint16')\n",
    "                    tifffile.imsave(os.path.join(savedir,name+'.tif'),img) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11",
   "metadata": {},
   "outputs": [],
   "source": [
    "(np.stack([im]*3)).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12",
   "metadata": {},
   "outputs": [],
   "source": [
    "getname(mask_names[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13",
   "metadata": {},
   "outputs": [],
   "source": [
    "mask_names[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14",
   "metadata": {},
   "outputs": [],
   "source": [
    "t = io.imread('/home/kcutler/DataDrive/omnipose_all/morphometrics_eval/5I_crop_ensemble_1_Gparent.tif')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16",
   "metadata": {},
   "outputs": [],
   "source": [
    "basedir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyqtgraph as pg\n",
    "pg.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "18",
   "metadata": {},
   "outputs": [],
   "source": [
    "import mahotas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19",
   "metadata": {},
   "outputs": [],
   "source": [
    "mahotas.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21",
   "metadata": {},
   "outputs": [],
   "source": [
    "numpy.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}