RadiologyCLIP.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ayyucedemirbas/RadiologyCLIP/blob/main/RadiologyCLIP.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import random\n",
        "import json\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras import layers, models\n",
        "from tensorflow.keras.preprocessing.text import Tokenizer\n",
        "from tensorflow.keras.preprocessing.image import load_img, img_to_array"
      ],
      "metadata": {
        "id": "HTtaRbEhSzv2"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!wget https://openi.nlm.nih.gov/imgs/collections/NLMCXR_png.tgz"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uTgnbLK7S2Wi",
        "outputId": "ee6a2c21-025c-4dd6-b49b-51fb0c4652a1"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2025-02-01 17:53:08--  https://openi.nlm.nih.gov/imgs/collections/NLMCXR_png.tgz\n",
            "Resolving openi.nlm.nih.gov (openi.nlm.nih.gov)... 130.14.65.157, 2607:f220:41e:7065::157\n",
            "Connecting to openi.nlm.nih.gov (openi.nlm.nih.gov)|130.14.65.157|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 1360814128 (1.3G) [application/x-gzip]\n",
            "Saving to: ‘NLMCXR_png.tgz’\n",
            "\n",
            "NLMCXR_png.tgz      100%[===================>]   1.27G  1.99MB/s    in 11m 0s  \n",
            "\n",
            "2025-02-01 18:04:08 (1.97 MB/s) - ‘NLMCXR_png.tgz’ saved [1360814128/1360814128]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!wget https://openi.nlm.nih.gov/imgs/collections/NLMCXR_reports.tgz"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Kf5uaJSHS-2W",
        "outputId": "d2c983c4-d9c6-40ea-b9fc-5beb0967cc54"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2025-02-01 18:04:27--  https://openi.nlm.nih.gov/imgs/collections/NLMCXR_reports.tgz\n",
            "Resolving openi.nlm.nih.gov (openi.nlm.nih.gov)... 130.14.65.157, 2607:f220:41e:7065::157\n",
            "Connecting to openi.nlm.nih.gov (openi.nlm.nih.gov)|130.14.65.157|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 1112632 (1.1M) [application/x-gzip]\n",
            "Saving to: ‘NLMCXR_reports.tgz’\n",
            "\n",
            "NLMCXR_reports.tgz  100%[===================>]   1.06M  1.69MB/s    in 0.6s    \n",
            "\n",
            "2025-02-01 18:04:28 (1.69 MB/s) - ‘NLMCXR_reports.tgz’ saved [1112632/1112632]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!mkdir workdir"
      ],
      "metadata": {
        "id": "0JeibBNWekj2"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%cd workdir"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZlruHtxeeme3",
        "outputId": "82020089-29eb-44e0-f2a5-ebe6beddb8f8"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/workdir\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!tar -xvzf /content/NLMCXR_png.tgz -C .\n",
        "!tar -xvzf /content/NLMCXR_reports.tgz -C ."
      ],
      "metadata": {
        "id": "Egz8vlPCihAa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%cd .."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JUWF8ag4ijl6",
        "outputId": "2e830b7e-bcdd-4ea6-af88-5645aa00797f"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!mv workdir/ecgen-radiology ."
      ],
      "metadata": {
        "id": "KKGa09tLiyJT"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!ls -al"
      ],
      "metadata": {
        "id": "xa9oVr5_i1x1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2080fe55-f815-4f50-dbed-3fb5bf25e85b"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "total 1330472\n",
            "drwxr-xr-x 1 root root       4096 Feb  1 18:05 .\n",
            "drwxr-xr-x 1 root root       4096 Feb  1 17:48 ..\n",
            "drwxr-xr-x 4 root root       4096 Jan 30 14:18 .config\n",
            "drwxr-xr-x 2  929  212      94208 Feb  4  2016 ecgen-radiology\n",
            "-rw-r--r-- 1 root root 1360814128 Sep  6  2016 NLMCXR_png.tgz\n",
            "-rw-r--r-- 1 root root    1112632 Jun  6  2017 NLMCXR_reports.tgz\n",
            "drwxr-xr-x 1 root root       4096 Jan 30 14:19 sample_data\n",
            "drwxr-xr-x 2   48   48     352256 Feb  1 18:05 workdir\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import xml.etree.ElementTree as ET"
      ],
      "metadata": {
        "id": "O0-d4j20inSA"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def find_valid_pairs(image_dir='workdir', report_dir='ecgen-radiology'):\n",
        "    valid_pairs = []\n",
        "    report_files = {}\n",
        "\n",
        "    for root, _, files in os.walk(report_dir):\n",
        "        for f in files:\n",
        "            if f.lower().endswith('.xml'):\n",
        "                base_name = os.path.splitext(f)[0].lower()\n",
        "                report_files[base_name] = os.path.join(root, f)\n",
        "\n",
        "    for root, _, files in os.walk(image_dir):\n",
        "        for f in files:\n",
        "            if f.lower().endswith('.png'):\n",
        "                base_name = os.path.splitext(f)[0].lower()\n",
        "                if base_name in report_files:\n",
        "                    valid_pairs.append((\n",
        "                        os.path.join(root, f),\n",
        "                        report_files[base_name]\n",
        "                    ))\n",
        "    return valid_pairs\n",
        "\n",
        "\n",
        "def parse_xml_report(xml_path):\n",
        "    try:\n",
        "        tree = ET.parse(xml_path)\n",
        "        root = tree.getroot()\n",
        "        ns = {'pmc': 'http://www.ncbi.nlm.nih.gov/pmc/articles/PMC'}\n",
        "\n",
        "        findings_elem = root.find(\".//pmc:AbstractText[@Label='FINDINGS']\", ns)\n",
        "        impression_elem = root.find(\".//pmc:AbstractText[@Label='IMPRESSION']\", ns)\n",
        "\n",
        "        findings = findings_elem.text.strip() if (findings_elem is not None and findings_elem.text) else \"\"\n",
        "        impression = impression_elem.text.strip() if (impression_elem is not None and impression_elem.text) else \"\"\n",
        "\n",
        "        if not findings and not impression:\n",
        "            return None\n",
        "\n",
        "        return f\"FINDINGS: {findings}. IMPRESSION: {impression}\"\n",
        "\n",
        "    except Exception as e:\n",
        "        print(f\"Error parsing {xml_path}: {str(e)}\")\n",
        "        return None\n"
      ],
      "metadata": {
        "id": "Ka7j4WAni3yp"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import xml.etree.ElementTree as ET\n",
        "import pandas as pd\n",
        "\n",
        "def parse_xml_report(xml_path):\n",
        "    try:\n",
        "        tree = ET.parse(xml_path)\n",
        "        root = tree.getroot()\n",
        "        ns = {'pmc': 'http://www.ncbi.nlm.nih.gov/pmc/articles/PMC'}\n",
        "\n",
        "        findings = root.find(\".//AbstractText[@Label='FINDINGS']\", ns)\n",
        "        impression = root.find(\".//AbstractText[@Label='IMPRESSION']\", ns)\n",
        "\n",
        "        findings_text = findings.text.strip() if (findings is not None and findings.text) else \"\"\n",
        "        impression_text = impression.text.strip() if (impression is not None and impression.text) else \"\"\n",
        "\n",
        "        if not findings_text and not impression_text:\n",
        "            return None, []\n",
        "\n",
        "        image_ids = [img.get('id') for img in root.findall('.//parentImage')]\n",
        "\n",
        "        image_paths = [f\"workdir/{img_id}.png\" for img_id in image_ids]\n",
        "\n",
        "        return (\n",
        "            f\"FINDINGS: {findings_text}. IMPRESSION: {impression_text}\",\n",
        "            image_paths\n",
        "        )\n",
        "\n",
        "    except Exception as e:\n",
        "        print(f\"Error parsing {xml_path}: {str(e)}\")\n",
        "        return None, []\n",
        "\n",
        "def create_dataset():\n",
        "    data = []\n",
        "\n",
        "    for root_dir, _, files in os.walk(\"ecgen-radiology\"):\n",
        "        for file in files:\n",
        "            if file.lower().endswith('.xml'):\n",
        "                xml_path = os.path.join(root_dir, file)\n",
        "                report_text, image_paths = parse_xml_report(xml_path)\n",
        "\n",
        "                if report_text and image_paths:\n",
        "                    for img_path in image_paths:\n",
        "                        if os.path.exists(img_path):\n",
        "                            data.append({\n",
        "                                'image_path': img_path,\n",
        "                                'report': report_text\n",
        "                            })\n",
        "                        else:\n",
        "                            print(f\"Missing image: {img_path}\")\n",
        "\n",
        "    return pd.DataFrame(data)\n",
        "\n",
        "radiology_df = create_dataset()\n",
        "print(f\"Found {len(radiology_df)} valid image-report pairs\")\n",
        "\n",
        "if not radiology_df.empty:\n",
        "    print(\"\\nSample entry:\")\n",
        "    print(f\"Image path: {radiology_df.iloc[0]['image_path']}\")\n",
        "    print(f\"Report: {radiology_df.iloc[0]['report']}\")\n",
        "else:\n",
        "    print(\"\\nNo valid pairs found. Check:\")\n",
        "    print(\"- XML files in ecgen-radiology directory\")\n",
        "    print(\"- Image files in workdir directory\")\n",
        "    print(\"- File naming consistency between XML and PNG files\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Yjyjyp2HjClS",
        "outputId": "f38c9359-2748-49d8-bda5-9071359b4298"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Found 7430 valid image-report pairs\n",
            "\n",
            "Sample entry:\n",
            "Image path: workdir/CXR1329_IM-0211-1001.png\n",
            "Report: FINDINGS: Two nodules are noted in the right XXXX XXXX measuring 13 mm and one measuring 16 mm in diameter. The smaller one appears to be within the right upper lobe and the large XXXX appears to be within the left lower lobe. No focal consolidation and no other pulmonary nodules are identified. However, if a full evaluation for lung nodules is desired consider XXXX for further evaluation. No pleural effusions or pneumothoraces. Heart and mediastinum of normal size and contour.. IMPRESSION: At XXXX 2 right lung pulmonary nodules concerning for<BR>metastatic disease\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = Tokenizer(\n",
        "    num_words=5000,\n",
        "    oov_token=\"\",\n",
        "    filters='!\"#$%&()*+.,-/:;=?@[\\]^_`{|}~ '\n",
        ")\n",
        "\n",
        "\n",
        "tokenizer.fit_on_texts(radiology_df['report'])\n",
        "vocab_size = len(tokenizer.word_index) + 1\n",
        "max_length = 100\n",
        "\n",
        "\n",
        "print([word for word in tokenizer.word_index if \"pneumonia\" in word or \"effusion\" in word][:10])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oYb2IL0AjLF7",
        "outputId": "f3895f8b-cda2-477c-f458-f41433c07c43"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['effusion', 'effusions', 'pneumonia']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def preprocess_sample(sample):\n",
        "    img = load_img(sample['image_path'], target_size=(224, 224))\n",
        "    img = img_to_array(img) / 255.0\n",
        "\n",
        "    seq = tokenizer.texts_to_sequences([sample['report']])[0]\n",
        "    seq = tf.keras.preprocessing.sequence.pad_sequences([seq], maxlen=max_length, padding='post')[0]\n",
        "    return img, seq\n",
        "\n",
        "def dataset_generator(df):\n",
        "    for idx, row in df.iterrows():\n",
        "        yield {'image_path': row['image_path'], 'report': row['report']}"
      ],
      "metadata": {
        "id": "jU7ZBCnNkA2i"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "output_signature = {\n",
        "    'image_path': tf.TensorSpec(shape=(), dtype=tf.string),\n",
        "    'report': tf.TensorSpec(shape=(), dtype=tf.string)\n",
        "}\n",
        "\n",
        "ds = tf.data.Dataset.from_generator(\n",
        "    lambda: dataset_generator(radiology_df),\n",
        "    output_signature=output_signature\n",
        ")"
      ],
      "metadata": {
        "id": "_D9vjde0kCah"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def map_func(image_path, report):\n",
        "    image_path = image_path.numpy().decode('utf-8')\n",
        "    report = report.numpy().decode('utf-8')\n",
        "    img, seq = preprocess_sample({'image_path': image_path, 'report': report})\n",
        "    return img, seq\n",
        "\n",
        "def tf_map_func(image_path, report):\n",
        "    img, seq = tf.py_function(\n",
        "        func=map_func,\n",
        "        inp=[image_path, report],\n",
        "        Tout=[tf.float32, tf.int32]\n",
        "    )\n",
        "    img.set_shape((224, 224, 3))\n",
        "    seq.set_shape((max_length,))\n",
        "    return img, seq"
      ],
      "metadata": {
        "id": "PdbwBiS0kHhC"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "batch_size = 16\n",
        "AUTOTUNE = tf.data.AUTOTUNE"
      ],
      "metadata": {
        "id": "G1d7uEcmkJJg"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "ds = ds.shuffle(buffer_size=len(radiology_df))\n",
        "ds = ds.map(lambda sample: tf_map_func(sample['image_path'], sample['report']),\n",
        "            num_parallel_calls=AUTOTUNE)\n",
        "ds = ds.batch(batch_size)\n",
        "ds = ds.prefetch(AUTOTUNE)"
      ],
      "metadata": {
        "id": "42dpQPw7kLGV"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "embedding_dim = 256\n",
        "num_transformer_blocks = 4\n",
        "num_heads = 8\n",
        "ff_dim = 512\n",
        "image_size = 224"
      ],
      "metadata": {
        "id": "7VP1KQfpkjEQ"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def create_vit_encoder(image_size):\n",
        "    inputs = layers.Input(shape=(image_size, image_size, 3))\n",
        "    patch_size = 16\n",
        "    num_patches = (image_size // patch_size) ** 2\n",
        "    projection_dim = 768\n",
        "\n",
        "    patches = layers.Conv2D(\n",
        "        filters=projection_dim,\n",
        "        kernel_size=patch_size,\n",
        "        strides=patch_size,\n",
        "        padding=\"valid\"\n",
        "    )(inputs)\n",
        "    patches = layers.Reshape((num_patches, projection_dim))(patches)\n",
        "\n",
        "    positional_embedding = layers.Embedding(input_dim=num_patches, output_dim=projection_dim)\n",
        "    positions = tf.range(start=0, limit=num_patches, delta=1)\n",
        "    encoded_patches = patches + positional_embedding(positions)\n",
        "\n",
        "    for _ in range(num_transformer_blocks):\n",
        "        x1 = layers.LayerNormalization(epsilon=1e-6)(encoded_patches)\n",
        "        attention_output = layers.MultiHeadAttention(\n",
        "            num_heads=num_heads, key_dim=projection_dim)(x1, x1)\n",
        "        x2 = layers.Add()([x1, attention_output])\n",
        "        x3 = layers.LayerNormalization(epsilon=1e-6)(x2)\n",
        "        ffn_output = layers.Dense(ff_dim, activation=\"relu\")(x3)\n",
        "        ffn_output = layers.Dense(projection_dim)(ffn_output)\n",
        "        encoded_patches = layers.Add()([x2, ffn_output])\n",
        "\n",
        "    model = models.Model(inputs, encoded_patches, name=\"vit_encoder\")\n",
        "    return model\n",
        "\n",
        "def create_image_encoder(image_size, embedding_dim):\n",
        "    inputs = layers.Input(shape=(image_size, image_size, 3))\n",
        "    vit = create_vit_encoder(image_size)\n",
        "    x = vit(inputs)\n",
        "    x = layers.GlobalAveragePooling1D()(x)\n",
        "    # Project to shared embedding dimension\n",
        "    outputs = layers.Dense(embedding_dim)(x)\n",
        "    model = models.Model(inputs, outputs, name=\"image_encoder\")\n",
        "    return model\n",
        "\n",
        "def create_text_encoder(vocab_size, embedding_dim, max_length):\n",
        "    inputs = layers.Input(shape=(max_length,))\n",
        "    x = layers.Embedding(vocab_size, embedding_dim, mask_zero=True)(inputs)\n",
        "\n",
        "    positional_embeddings = layers.Embedding(max_length, embedding_dim)(tf.range(start=0, limit=max_length, delta=1))\n",
        "    x = x + positional_embeddings\n",
        "\n",
        "    for _ in range(num_transformer_blocks):\n",
        "        x1 = layers.LayerNormalization(epsilon=1e-6)(x)\n",
        "        attention_output = layers.MultiHeadAttention(\n",
        "            num_heads=num_heads, key_dim=embedding_dim)(x1, x1)\n",
        "        x2 = layers.Add()([x1, attention_output])\n",
        "        x3 = layers.LayerNormalization(epsilon=1e-6)(x2)\n",
        "        ffn_output = layers.Dense(ff_dim, activation=\"relu\")(x3)\n",
        "        ffn_output = layers.Dense(embedding_dim)(ffn_output)\n",
        "        x = layers.Add()([x2, ffn_output])\n",
        "    x = layers.GlobalAveragePooling1D()(x)\n",
        "    outputs = layers.Dense(embedding_dim)(x)\n",
        "    model = models.Model(inputs, outputs, name=\"text_encoder\")\n",
        "    return model\n"
      ],
      "metadata": {
        "id": "eYnUP8LfkoDA"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "image_encoder = create_image_encoder(image_size, embedding_dim)\n",
        "text_encoder = create_text_encoder(vocab_size, embedding_dim, max_length)"
      ],
      "metadata": {
        "id": "0gwCDiuWkqC4"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class CLIPModel(tf.keras.Model):\n",
        "    def __init__(self, image_encoder, text_encoder, temperature=0.05):\n",
        "        super(CLIPModel, self).__init__()\n",
        "        self.image_encoder = image_encoder\n",
        "        self.text_encoder = text_encoder\n",
        "        self.temperature = tf.Variable(temperature, trainable=True, dtype=tf.float32)\n",
        "\n",
        "    def compile(self, optimizer):\n",
        "        super(CLIPModel, self).compile()\n",
        "        self.optimizer = optimizer\n",
        "        self.loss_tracker = tf.keras.metrics.Mean(name=\"loss\")\n",
        "\n",
        "    @property\n",
        "    def metrics(self):\n",
        "        return [self.loss_tracker]\n",
        "\n",
        "    def train_step(self, data):\n",
        "        images, texts = data  # images: (batch, H, W, 3); texts: (batch, max_length)\n",
        "\n",
        "        with tf.GradientTape() as tape:\n",
        "            img_embeddings = self.image_encoder(images, training=True)\n",
        "            txt_embeddings = self.text_encoder(texts, training=True)\n",
        "\n",
        "            img_embeddings = tf.math.l2_normalize(img_embeddings, axis=1)\n",
        "            txt_embeddings = tf.math.l2_normalize(txt_embeddings, axis=1)\n",
        "\n",
        "            # Compute similarity logits: (batch, batch)\n",
        "            logits = tf.matmul(img_embeddings, txt_embeddings, transpose_b=True)\n",
        "            logits = logits / self.temperature\n",
        "\n",
        "            # Ground truth: diagonal elements are the matching pairs.\n",
        "            batch_size = tf.shape(images)[0]\n",
        "            labels = tf.range(batch_size)\n",
        "\n",
        "            # Compute cross-entropy loss for image->text and text->image\n",
        "            loss_i2t = tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)\n",
        "            loss_t2i = tf.keras.losses.sparse_categorical_crossentropy(labels, tf.transpose(logits), from_logits=True)\n",
        "            loss = (loss_i2t + loss_t2i) / 2.0\n",
        "            # implemented indirectly via cross-entropy on the similarity logits\n",
        "\n",
        "        # Compute gradients and update weights\n",
        "        gradients = tape.gradient(loss, self.trainable_variables)\n",
        "        self.optimizer.apply_gradients(zip(gradients, self.trainable_variables))\n",
        "        self.loss_tracker.update_state(loss)\n",
        "        return {\"loss\": self.loss_tracker.result()}\n",
        "\n",
        "    def call(self, inputs):\n",
        "        images, texts = inputs\n",
        "        img_embeddings = tf.math.l2_normalize(self.image_encoder(images, training=False), axis=1)\n",
        "        txt_embeddings = tf.math.l2_normalize(self.text_encoder(texts, training=False), axis=1)\n",
        "        return img_embeddings, txt_embeddings"
      ],
      "metadata": {
        "id": "bFEfSXFhkuoq"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "clip_model = CLIPModel(image_encoder, text_encoder, temperature=0.05)\n",
        "optimizer = tf.keras.optimizers.Adam(learning_rate=1e-4)\n",
        "clip_model.compile(optimizer=optimizer)"
      ],
      "metadata": {
        "id": "Ps6-9P5nkw_S"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "epochs = 10 #you can increase it\n",
        "clip_model.fit(ds, epochs=epochs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "y7_LT3KSk0Mp",
        "outputId": "81ce59b8-7e66-49f8-d873-cd2f63a43f57"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/10\n",
            "\u001b[1m465/465\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m435s\u001b[0m 929ms/step - loss: 2.7765\n",
            "Epoch 2/10\n",
            "\u001b[1m465/465\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m440s\u001b[0m 925ms/step - loss: 2.7726\n",
            "Epoch 3/10\n",
            "\u001b[1m465/465\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m442s\u001b[0m 925ms/step - loss: 2.7726\n",
            "Epoch 4/10\n",
            "\u001b[1m465/465\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m442s\u001b[0m 926ms/step - loss: 2.7726\n",
            "Epoch 5/10\n",
            "\u001b[1m465/465\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m442s\u001b[0m 925ms/step - loss: 2.7726\n",
            "Epoch 6/10\n",
            "\u001b[1m465/465\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m441s\u001b[0m 924ms/step - loss: 2.7726\n",
            "Epoch 7/10\n",
            "\u001b[1m465/465\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m442s\u001b[0m 925ms/step - loss: 2.7726\n",
            "Epoch 8/10\n",
            "\u001b[1m465/465\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m442s\u001b[0m 925ms/step - loss: 2.7726\n",
            "Epoch 9/10\n",
            "\u001b[1m465/465\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m443s\u001b[0m 928ms/step - loss: 2.7726\n",
            "Epoch 10/10\n",
            "\u001b[1m465/465\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m442s\u001b[0m 928ms/step - loss: 2.7726\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.src.callbacks.history.History at 0x7eee8446d310>"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def get_text_embedding(text):\n",
        "    seq = tokenizer.texts_to_sequences([text])[0]\n",
        "    seq = tf.keras.preprocessing.sequence.pad_sequences([seq], maxlen=max_length, padding='post')\n",
        "    emb = text_encoder(seq, training=False)\n",
        "    emb = tf.math.l2_normalize(emb, axis=1)\n",
        "    return emb\n",
        "\n",
        "def get_image_embedding(image_path):\n",
        "    img = load_img(image_path, target_size=(image_size, image_size))\n",
        "    img = img_to_array(img) / 255.0\n",
        "    img = np.expand_dims(img, axis=0)\n",
        "    emb = image_encoder(img, training=False)\n",
        "    emb = tf.math.l2_normalize(emb, axis=1)\n",
        "    return emb\n",
        "\n",
        "test_image = 'xr.jpeg'\n",
        "img_emb = get_image_embedding(test_image)\n",
        "print(\"Image embedding:\", img_emb.numpy())\n",
        "\n",
        "all_reports = radiology_df['report'].tolist()\n",
        "all_text_embeddings = []\n",
        "for report in all_reports:\n",
        "    emb = get_text_embedding(report)\n",
        "    all_text_embeddings.append(emb.numpy()[0])\n",
        "all_text_embeddings = np.array(all_text_embeddings)\n",
        "\n",
        "cosine_sim = np.dot(all_text_embeddings, img_emb.numpy()[0])\n",
        "best_idx = np.argmax(cosine_sim)\n",
        "print(\"\\nBest matching report:\")\n",
        "print(all_reports[best_idx])"
      ],
      "metadata": {
        "id": "ak9AcSOLj9pc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ee82119b-508d-4ec2-be61-1fc745c3655a"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Image embedding: [[ 0.05733508  0.00523869  0.06383529  0.05051493  0.04355091  0.02500839\n",
            "  -0.07777718  0.1457428  -0.05198224  0.07125071 -0.0904338   0.06187361\n",
            "  -0.09715363  0.12147623 -0.01999591  0.03804174 -0.03612426  0.1149757\n",
            "  -0.01391032 -0.00195657  0.1456474  -0.07698674 -0.04502937 -0.03379296\n",
            "   0.07548678  0.00504048  0.04394463  0.02888563 -0.03367941  0.08016434\n",
            "  -0.11808249  0.11248635  0.13601714  0.00332339  0.01640047  0.04144336\n",
            "   0.09689321  0.02218678  0.09288475  0.08762556  0.04284214 -0.04380634\n",
            "   0.00071143 -0.04711069 -0.02884681  0.02897729  0.01118722 -0.03120163\n",
            "   0.05351967 -0.04563073 -0.01525307  0.02657428  0.06119745 -0.07053658\n",
            "  -0.07705502 -0.01606986  0.08667487 -0.05638002  0.02364861 -0.10039265\n",
            "   0.03728671 -0.04434197 -0.0638153  -0.02112338 -0.0598181   0.09828798\n",
            "   0.07277232  0.01066731  0.04956194 -0.01133192 -0.00738006  0.01844435\n",
            "  -0.05654645  0.00830322 -0.07037811  0.03265889  0.0075481  -0.02276928\n",
            "   0.00311648 -0.04058895 -0.0764382  -0.07895491  0.03984361 -0.00759744\n",
            "   0.04593149  0.02649157  0.00983189 -0.02094586  0.01819062 -0.0601249\n",
            "   0.00920953  0.01981664 -0.13653192  0.03457103  0.00672623  0.02281417\n",
            "   0.05499525  0.14945713 -0.08611275  0.03455135  0.00442015 -0.1687858\n",
            "  -0.1420991  -0.02390151  0.05159623  0.03153037 -0.08091021  0.0858635\n",
            "  -0.02496115 -0.03190533  0.01997001  0.00884347  0.01857662  0.01853621\n",
            "   0.03872626  0.05767087 -0.04467797 -0.01404961 -0.09870126  0.02899238\n",
            "   0.01292208 -0.0358163  -0.0237358   0.00080296 -0.06264865 -0.03244296\n",
            "  -0.01102658 -0.12434643  0.03842926  0.03723353 -0.00224009 -0.11911049\n",
            "  -0.04859433  0.09091132 -0.08951554  0.15156688  0.09286171  0.01512262\n",
            "  -0.02501453 -0.03349034  0.03914319 -0.10845304  0.08299521  0.087121\n",
            "  -0.02152796  0.08331224  0.01688227  0.03791194  0.06249746  0.00722949\n",
            "  -0.06462058  0.03613601  0.10156877 -0.0847126  -0.07610279 -0.08201543\n",
            "   0.03576325  0.01397955  0.05651496  0.05388753 -0.06828603 -0.03182669\n",
            "   0.05567235  0.01083691  0.02447353  0.06992363  0.04895486  0.1132594\n",
            "  -0.05656235 -0.04235588  0.03229396 -0.0799403   0.04146328  0.05818685\n",
            "   0.09530976  0.02661592  0.0270949   0.00142315 -0.08882003  0.00251872\n",
            "   0.0642683  -0.03921587 -0.05825542  0.00260841 -0.0737768  -0.03079709\n",
            "   0.04956103  0.10231739  0.03201401 -0.05698542 -0.00446673  0.00719716\n",
            "  -0.00669434  0.02686224 -0.00618782  0.05757782  0.0789791   0.04005319\n",
            "   0.07323772 -0.00033798  0.0466357   0.07474092  0.01124983  0.06368975\n",
            "   0.01774672 -0.03479566  0.04445657 -0.05743926 -0.02078703  0.01674717\n",
            "  -0.01740522 -0.08757166 -0.14845812 -0.03553953  0.03221298  0.03464089\n",
            "  -0.0285788  -0.07815813 -0.07398876  0.08625674 -0.03380406  0.00498513\n",
            "   0.12473664 -0.01818262  0.00696791  0.01527237 -0.00414472  0.03493051\n",
            "   0.02009369 -0.07285822  0.11607243 -0.02455128 -0.04456853 -0.09075557\n",
            "   0.01863879  0.04617385 -0.0237237  -0.03285275 -0.04249278 -0.18154396\n",
            "  -0.04049987 -0.02700712 -0.04692007  0.00752405 -0.06931389 -0.02956312\n",
            "  -0.02692176  0.0610798   0.04395137  0.0329669   0.02495447 -0.10759334\n",
            "   0.15546478  0.05334622  0.12364908  0.03305594]]\n",
            "\n",
            "Best matching report:\n",
            "FINDINGS: . IMPRESSION: 13 - 14 mm ring-shaped opacity with central lucency, seen with certainty on PA view only, projects over right midlung. Uncertain if this represent something within the lung or external to the patient. There is also a 5 mm nodular opacity projecting over the left upper lung at the level of AP XXXX. These findings could be further evaluated XXXX if clinically indicated. No acute airspace disease. Mediastinal contour normal limits. No effusions.\n"
          ]
        }
      ]
    }
  ]
}