FIxes for batch prompt creation

Author: ashwinprasadme

extras/openai_batch_inference.ipynb

@@ -0,0 +1,136 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "https://platform.openai.com/docs/guides/batch/model-availability"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from openai import OpenAI\n",
+    "import os\n",
+    "import yaml\n",
+    "\n",
+    "# os.environ.get(\"OPENAI_API_KEY\")\n",
+    "# Get config\n",
+    "CONFIG_FILE = \"../src/config.yaml\"\n",
+    "\n",
+    "if os.path.exists(CONFIG_FILE):\n",
+    "    with open(CONFIG_FILE, \"r\") as file:\n",
+    "        config = yaml.safe_load(file)\n",
+    "else:\n",
+    "    config = {}\n",
+    "\n",
+    "os.environ[\"OPENAI_API_KEY\"] = config[\"llm\"][\"openai_key\"]\n",
+    "client = OpenAI()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "file_path = \"../.scrapy/results/gpt-4o-mini/autogen_typeevalpy_benchmark/batch_prompt.jsonl\"\n",
+    "\n",
+    "batch_input_file = client.files.create(\n",
+    "  file=open(file_path, \"rb\"),\n",
+    "  purpose=\"batch\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Batch(id='batch_Ufc5EkBcAbSceD4XzNF67KjO', completion_window='24h', created_at=1725549306, endpoint='/v1/chat/completions', input_file_id='file-PyayymAZ5TCcbTZUZ5nPN5zI', object='batch', status='validating', cancelled_at=None, cancelling_at=None, completed_at=None, error_file_id=None, errors=None, expired_at=None, expires_at=1725635706, failed_at=None, finalizing_at=None, in_progress_at=None, metadata={'description': 'Batch Trial'}, output_file_id=None, request_counts=BatchRequestCounts(completed=0, failed=0, total=0))"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "batch_input_file_id = batch_input_file.id\n",
+    "\n",
+    "client.batches.create(\n",
+    "    input_file_id=batch_input_file_id,\n",
+    "    endpoint=\"/v1/chat/completions\",\n",
+    "    completion_window=\"24h\",\n",
+    "    metadata={\n",
+    "      \"description\": \"Batch Trial\"\n",
+    "    }\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from openai import OpenAI\n",
+    "client = OpenAI()\n",
+    "\n",
+    "file_info = client.batches.retrieve(\"batch_Ufc5EkBcAbSceD4XzNF67KjO\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{\"id\": \"batch_req_HWbEoplsNE1CoXKmYvRKUxK0\", \"custom_id\": \"request-types-0\", \"response\": {\"status_code\": 200, \"request_id\": \"e55227a57343f3bfedc16b45f392647e\", \"body\": {\"id\": \"chatcmpl-A48UNoLfHd5nmm1SRA3E3GEEH0RNc\", \"object\": \"chat.completion\", \"created\": 1725549307, \"model\": \"gpt-4o-mini-2024-07-18\", \"choices\": [{\"index\": 0, \"message\": {\"role\": \"assistant\", \"content\": \"1. int  \\n2. int  \\n3. function  \\n4. function  \\n5. int  \", \"refusal\": null}, \"logprobs\": null, \"finish_reason\": \"stop\"}], \"usage\": {\"prompt_tokens\": 357, \"completion_tokens\": 20, \"total_tokens\": 377}, \"system_fingerprint\": \"fp_f33667828e\"}}, \"error\": null}\n",
+      "{\"id\": \"batch_req_GdlSnHJQ0dBcoL8SKDoP9d0F\", \"custom_id\": \"request-types-1\", \"response\": {\"status_code\": 200, \"request_id\": \"9e01043de3caf7e095f7d799c7f3279b\", \"body\": {\"id\": \"chatcmpl-A48UNq9mYTJbpRYQJQNfPdyFu4Whj\", \"object\": \"chat.completion\", \"created\": 1725549307, \"model\": \"gpt-4o-mini-2024-07-18\", \"choices\": [{\"index\": 0, \"message\": {\"role\": \"assistant\", \"content\": \"1. float  \\n2. float  \\n3. function  \\n4. function  \\n5. float  \", \"refusal\": null}, \"logprobs\": null, \"finish_reason\": \"stop\"}], \"usage\": {\"prompt_tokens\": 359, \"completion_tokens\": 20, \"total_tokens\": 379}, \"system_fingerprint\": \"fp_f33667828e\"}}, \"error\": null}\n",
+      "{\"id\": \"batch_req_05mDnLSDPh9f3cO5BDtuEnzJ\", \"custom_id\": \"request-types-2\", \"response\": {\"status_code\": 200, \"request_id\": \"54eda3aa77bc48b7ebf99431901c86f7\", \"body\": {\"id\": \"chatcmpl-A48UNwnSBiX2lmAbptZz4yyUMQgrp\", \"object\": \"chat.completion\", \"created\": 1725549307, \"model\": \"gpt-4o-mini-2024-07-18\", \"choices\": [{\"index\": 0, \"message\": {\"role\": \"assistant\", \"content\": \"1. str  \\n2. str  \\n3. function  \\n4. function  \\n5. str  \", \"refusal\": null}, \"logprobs\": null, \"finish_reason\": \"stop\"}], \"usage\": {\"prompt_tokens\": 359, \"completion_tokens\": 20, \"total_tokens\": 379}, \"system_fingerprint\": \"fp_5bd87c427a\"}}, \"error\": null}\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "from openai import OpenAI\n",
+    "client = OpenAI()\n",
+    "\n",
+    "file_response = client.files.content(file_info.output_file_id)\n",
+    "print(file_response.text)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

requirements.txt

@@ -7,4 +7,6 @@ docker==6.1.3
 tabulate==0.9.0
 PyYAML
 requests==2.31.0
-tqdm
+tqdm
+tiktoken
+openai

src/target_tools/llms/requirements.txt

@@ -12,4 +12,5 @@ packaging
 peft
 torch==2.1.2
 torchvision==0.16.2
-torchaudio
+torchaudio
+tiktoken

src/target_tools/llms/src/runner.py

@@ -201,7 +201,11 @@ def model_evaluation_openai(
 
     utils.get_prompt_cost(prompts)
     utils.dump_ft_jsonl(id_mapping, f"{results_dst}/ft_dataset.jsonl")
-    utils.dump_batch_prompt_jsonl(id_mapping, f"{results_dst}/batch_prompt.jsonl")
+    utils.dump_batch_prompt_jsonl(
+        id_mapping,
+        f"{results_dst}/batch_prompt.jsonl",
+        model=model_name,
+    )
 
     request_outputs = openai_helpers.process_requests(
         model_name,

src/target_tools/llms/src/utils.py

@@ -266,7 +266,6 @@ def get_prompt(prompt_id, file_path, answers_placeholders=True, use_system_promp
 def dump_ft_jsonl(id_mapping, output_file):
     mappings = copy.deepcopy(id_mapping)
     for _m in mappings.values():
-        print(_m)
         assistant_message = {
             "role": "assistant",
             "content": generate_answers_for_fine_tuning(_m["json_filepath"]),
@@ -281,12 +280,22 @@ def dump_ft_jsonl(id_mapping, output_file):
             output.write("\n")
 
 
-def dump_batch_prompt_jsonl(id_mapping, output_file):
-    prompts = [x["prompt"] for x in id_mapping.values()]
-
+def dump_batch_prompt_jsonl(
+    id_mapping, output_file, id_prefix="types", model="gpt-4o-mini"
+):
     with open(output_file, "w") as output:
-        for _m in prompts:
-            output.write(json.dumps(_m))
+        for idx, _m in id_mapping.items():
+            prompt_dict = {
+                "custom_id": f"request-{id_prefix}-{idx}",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": model,
+                    "messages": _m["prompt"],
+                    "max_tokens": 250,
+                },
+            }
+            output.write(json.dumps(prompt_dict))
             output.write("\n")