Skip to content

Commit 98772e8

Browse files
ashwinprasadmeashwinprasadme
committed
Minor cleanup
1 parent cfedc1f commit 98772e8

File tree

8 files changed

+324
-69
lines changed

8 files changed

+324
-69
lines changed

‎extras/openai_batch_inference.ipynb

+128-22
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
},
1010
{
1111
"cell_type": "code",
12-
"execution_count": 7,
12+
"execution_count": 1,
1313
"metadata": {},
1414
"outputs": [],
1515
"source": [
@@ -33,46 +33,152 @@
3333
},
3434
{
3535
"cell_type": "code",
36-
"execution_count": 8,
36+
"execution_count": 3,
3737
"metadata": {},
38-
"outputs": [],
38+
"outputs": [
39+
{
40+
"name": "stdout",
41+
"output_type": "stream",
42+
"text": [
43+
"batch_prompt_gpt-4o-mini_autogen.jsonl\n",
44+
"batch_prompt_gpt-4o-mini_hg_cs.jsonl\n",
45+
"batch_prompt_gpt-4o-mini_js.jsonl\n",
46+
"batch_prompt_gpt-4o-mini_micro.jsonl\n",
47+
"batch_prompt_gpt-4o-mini_pycg.jsonl\n",
48+
"batch_prompt_gpt-4o_autogen.jsonl\n",
49+
"batch_prompt_gpt-4o_hg_cs.jsonl\n",
50+
"batch_prompt_gpt-4o_js.jsonl\n",
51+
"batch_prompt_gpt-4o_micro.jsonl\n",
52+
"batch_prompt_gpt-4o_pycg.jsonl\n"
53+
]
54+
}
55+
],
3956
"source": [
57+
"folder_path = \"../.scrapy/batch_prompts\"\n",
58+
"file_paths = []\n",
4059
"\n",
41-
"file_path = \"../.scrapy/results/gpt-4o-mini/autogen_typeevalpy_benchmark/batch_prompt.jsonl\"\n",
42-
"\n",
43-
"batch_input_file = client.files.create(\n",
44-
" file=open(file_path, \"rb\"),\n",
45-
" purpose=\"batch\"\n",
46-
")"
60+
"# Get the list of file paths in the folder\n",
61+
"for file_name in os.listdir(folder_path):\n",
62+
" if file_name.endswith(\".jsonl\"):\n",
63+
" file_paths.append(os.path.join(folder_path, file_name))\n",
64+
" print(file_name)"
65+
]
66+
},
67+
{
68+
"cell_type": "markdown",
69+
"metadata": {},
70+
"source": [
71+
"- batch_prompt_gpt-4o-mini_autogen.jsonl - 'file-IHVfDSrWeL8YerJ6wMZvtuol' - batch_glNBmyi30uDTH0EuG3c6jTov\n",
72+
"- batch_prompt_gpt-4o-mini_hg_cs.jsonl - 'file-rbzekY6CP6dLdjCLi91jfcXD' - batch_3dK2KOYlkhovzS7Qmqbs0Cnc\n",
73+
"- batch_prompt_gpt-4o-mini_js.jsonl - 'file-1yimk0k7iP7hH8Vbqr3KH2aD' - batch_rbNCMtcZIUyzD9aZ4QQ2RsIu\n",
74+
"- batch_prompt_gpt-4o-mini_micro.jsonl - 'file-Bt6l3A48l8QAWgwAf2D1X3Ja' - batch_zz7O1DOR6r7eVpvuvn8g6GgZ\n",
75+
"- batch_prompt_gpt-4o-mini_pycg.jsonl - 'file-vH3YgoTcegi16LobgPhQdFZv' - batch_Nk9np2B9rhzD3AaDgMZEU7oB\n",
76+
"- batch_prompt_gpt-4o_autogen.jsonl - 'file-HgcNUnBHVrsWuSTG9lMep0Jb' - batch_sjDtwbjl3IB6GRZg5XP4svqS\n",
77+
"- batch_prompt_gpt-4o_hg_cs.jsonl - 'file-9FQ5qqxnmxP8J5Mjllnlv4H8' - batch_1PSl4bmOdffJdyRraiOPbleP\n",
78+
"- batch_prompt_gpt-4o_js.jsonl - 'file-SZiKRW43fng8I7l6fU61kxDn' - batch_YE7JzcBsKLSeZ7CeoeGIkCdZ\n",
79+
"- batch_prompt_gpt-4o_micro.jsonl - 'file-hYV4LDTjVZ5n1vLOEdLDegRZ' - batch_GL3PSRMWW25MNowx8ERWCDhD\n",
80+
"- batch_prompt_gpt-4o_pycg.jsonl - 'file-gKOW2WSvepdu4RqlGCbrtdj6' - batch_uGBHKpb5oeIHVaBNXUQCdipr"
4781
]
4882
},
4983
{
5084
"cell_type": "code",
51-
"execution_count": 9,
85+
"execution_count": 2,
5286
"metadata": {},
5387
"outputs": [
5488
{
55-
"data": {
56-
"text/plain": [
57-
"Batch(id='batch_Ufc5EkBcAbSceD4XzNF67KjO', completion_window='24h', created_at=1725549306, endpoint='/v1/chat/completions', input_file_id='file-PyayymAZ5TCcbTZUZ5nPN5zI', object='batch', status='validating', cancelled_at=None, cancelling_at=None, completed_at=None, error_file_id=None, errors=None, expired_at=None, expires_at=1725635706, failed_at=None, finalizing_at=None, in_progress_at=None, metadata={'description': 'Batch Trial'}, output_file_id=None, request_counts=BatchRequestCounts(completed=0, failed=0, total=0))"
58-
]
59-
},
60-
"execution_count": 9,
61-
"metadata": {},
62-
"output_type": "execute_result"
89+
"name": "stdout",
90+
"output_type": "stream",
91+
"text": [
92+
"['file-IHVfDSrWeL8YerJ6wMZvtuol', 'file-rbzekY6CP6dLdjCLi91jfcXD', 'file-1yimk0k7iP7hH8Vbqr3KH2aD', 'file-Bt6l3A48l8QAWgwAf2D1X3Ja', 'file-vH3YgoTcegi16LobgPhQdFZv', 'file-HgcNUnBHVrsWuSTG9lMep0Jb', 'file-9FQ5qqxnmxP8J5Mjllnlv4H8', 'file-SZiKRW43fng8I7l6fU61kxDn', 'file-hYV4LDTjVZ5n1vLOEdLDegRZ', 'file-gKOW2WSvepdu4RqlGCbrtdj6']\n"
93+
]
6394
}
6495
],
6596
"source": [
66-
"batch_input_file_id = batch_input_file.id\n",
97+
"import os\n",
98+
"\n",
99+
"folder_path = \"../.scrapy/batch_prompts\"\n",
100+
"file_paths = []\n",
67101
"\n",
68-
"client.batches.create(\n",
102+
"# Get the list of file paths in the folder\n",
103+
"for file_name in os.listdir(folder_path):\n",
104+
" if file_name.endswith(\".jsonl\"):\n",
105+
" file_paths.append(os.path.join(folder_path, file_name))\n",
106+
"\n",
107+
"batch_input_file_ids = []\n",
108+
"# Iterate over the file paths\n",
109+
"for file_path in file_paths:\n",
110+
" batch_input_file = client.files.create(\n",
111+
" file=open(file_path, \"rb\"),\n",
112+
" purpose=\"batch\"\n",
113+
" )\n",
114+
" batch_input_file_ids.append(batch_input_file.id)\n",
115+
"\n",
116+
"print(batch_input_file_ids)"
117+
]
118+
},
119+
{
120+
"cell_type": "code",
121+
"execution_count": 3,
122+
"metadata": {},
123+
"outputs": [],
124+
"source": [
125+
"for batch_input_file_id in batch_input_file_ids:\n",
126+
" client.batches.create(\n",
69127
" input_file_id=batch_input_file_id,\n",
70128
" endpoint=\"/v1/chat/completions\",\n",
71129
" completion_window=\"24h\",\n",
72130
" metadata={\n",
73-
" \"description\": \"Batch Trial\"\n",
131+
" \"description\": f\"Chat completions for {batch_input_file_id}\"\n",
74132
" }\n",
75-
")"
133+
" )\n"
134+
]
135+
},
136+
{
137+
"cell_type": "code",
138+
"execution_count": 2,
139+
"metadata": {},
140+
"outputs": [],
141+
"source": [
142+
"from openai import OpenAI\n",
143+
"client = OpenAI()\n",
144+
"\n",
145+
"batches_info = client.batches.list(limit=10)"
146+
]
147+
},
148+
{
149+
"cell_type": "code",
150+
"execution_count": 4,
151+
"metadata": {},
152+
"outputs": [
153+
{
154+
"name": "stdout",
155+
"output_type": "stream",
156+
"text": [
157+
"Batch(id='batch_uGBHKpb5oeIHVaBNXUQCdipr', completion_window='24h', created_at=1725565304, endpoint='/v1/chat/completions', input_file_id='file-gKOW2WSvepdu4RqlGCbrtdj6', object='batch', status='completed', cancelled_at=None, cancelling_at=None, completed_at=1725565483, error_file_id=None, errors=None, expired_at=None, expires_at=1725651704, failed_at=None, finalizing_at=1725565473, in_progress_at=1725565305, metadata={'description': 'Chat completions for file-gKOW2WSvepdu4RqlGCbrtdj6'}, output_file_id='file-q8B4XM9JGox8eRfdCJW5OUyp', request_counts=BatchRequestCounts(completed=127, failed=0, total=127))\n",
158+
"Batch(id='batch_GL3PSRMWW25MNowx8ERWCDhD', completion_window='24h', created_at=1725565303, endpoint='/v1/chat/completions', input_file_id='file-hYV4LDTjVZ5n1vLOEdLDegRZ', object='batch', status='completed', cancelled_at=None, cancelling_at=None, completed_at=1725565485, error_file_id=None, errors=None, expired_at=None, expires_at=1725651703, failed_at=None, finalizing_at=1725565473, in_progress_at=1725565303, metadata={'description': 'Chat completions for file-hYV4LDTjVZ5n1vLOEdLDegRZ'}, output_file_id='file-ykm28abo3iP6WgDXRHOyn3mf', request_counts=BatchRequestCounts(completed=154, failed=0, total=154))\n",
159+
"Batch(id='batch_YE7JzcBsKLSeZ7CeoeGIkCdZ', completion_window='24h', created_at=1725565302, endpoint='/v1/chat/completions', input_file_id='file-SZiKRW43fng8I7l6fU61kxDn', object='batch', status='completed', cancelled_at=None, cancelling_at=None, completed_at=1725565475, error_file_id=None, errors=None, expired_at=None, expires_at=1725651702, failed_at=None, finalizing_at=1725565466, in_progress_at=1725565302, metadata={'description': 'Chat completions for file-SZiKRW43fng8I7l6fU61kxDn'}, output_file_id='file-opoaKhRoEGW7SbTL1jQaW8ET', request_counts=BatchRequestCounts(completed=127, failed=0, total=127))\n",
160+
"Batch(id='batch_1PSl4bmOdffJdyRraiOPbleP', completion_window='24h', created_at=1725565301, endpoint='/v1/chat/completions', input_file_id='file-9FQ5qqxnmxP8J5Mjllnlv4H8', object='batch', status='completed', cancelled_at=None, cancelling_at=None, completed_at=1725565482, error_file_id=None, errors=None, expired_at=None, expires_at=1725651701, failed_at=None, finalizing_at=1725565473, in_progress_at=1725565302, metadata={'description': 'Chat completions for file-9FQ5qqxnmxP8J5Mjllnlv4H8'}, output_file_id='file-yLjmwM8tnJ7m52Z0kUDSg0xb', request_counts=BatchRequestCounts(completed=123, failed=0, total=123))\n",
161+
"Batch(id='batch_sjDtwbjl3IB6GRZg5XP4svqS', completion_window='24h', created_at=1725565299, endpoint='/v1/chat/completions', input_file_id='file-HgcNUnBHVrsWuSTG9lMep0Jb', object='batch', status='completed', cancelled_at=None, cancelling_at=None, completed_at=1725567142, error_file_id=None, errors=None, expired_at=None, expires_at=1725651699, failed_at=None, finalizing_at=1725566565, in_progress_at=1725565303, metadata={'description': 'Chat completions for file-HgcNUnBHVrsWuSTG9lMep0Jb'}, output_file_id='file-m3nARpb58gf6iCQQHot7rIYB', request_counts=BatchRequestCounts(completed=5453, failed=0, total=5453))\n",
162+
"Batch(id='batch_Nk9np2B9rhzD3AaDgMZEU7oB', completion_window='24h', created_at=1725565299, endpoint='/v1/chat/completions', input_file_id='file-vH3YgoTcegi16LobgPhQdFZv', object='batch', status='completed', cancelled_at=None, cancelling_at=None, completed_at=1725565635, error_file_id=None, errors=None, expired_at=None, expires_at=1725651699, failed_at=None, finalizing_at=1725565626, in_progress_at=1725565299, metadata={'description': 'Chat completions for file-vH3YgoTcegi16LobgPhQdFZv'}, output_file_id='file-ax8bNm0VQJ8CuVze0TO4rPQQ', request_counts=BatchRequestCounts(completed=127, failed=0, total=127))\n",
163+
"Batch(id='batch_zz7O1DOR6r7eVpvuvn8g6GgZ', completion_window='24h', created_at=1725565298, endpoint='/v1/chat/completions', input_file_id='file-Bt6l3A48l8QAWgwAf2D1X3Ja', object='batch', status='completed', cancelled_at=None, cancelling_at=None, completed_at=1725565638, error_file_id=None, errors=None, expired_at=None, expires_at=1725651698, failed_at=None, finalizing_at=1725565626, in_progress_at=1725565299, metadata={'description': 'Chat completions for file-Bt6l3A48l8QAWgwAf2D1X3Ja'}, output_file_id='file-5wER4twuGj6HoWiwDZPnhxIe', request_counts=BatchRequestCounts(completed=154, failed=0, total=154))\n",
164+
"Batch(id='batch_rbNCMtcZIUyzD9aZ4QQ2RsIu', completion_window='24h', created_at=1725565298, endpoint='/v1/chat/completions', input_file_id='file-1yimk0k7iP7hH8Vbqr3KH2aD', object='batch', status='completed', cancelled_at=None, cancelling_at=None, completed_at=1725565635, error_file_id=None, errors=None, expired_at=None, expires_at=1725651698, failed_at=None, finalizing_at=1725565626, in_progress_at=1725565298, metadata={'description': 'Chat completions for file-1yimk0k7iP7hH8Vbqr3KH2aD'}, output_file_id='file-bvZ1OLmpbIPTDkIrMN2EqAAY', request_counts=BatchRequestCounts(completed=127, failed=0, total=127))\n",
165+
"Batch(id='batch_3dK2KOYlkhovzS7Qmqbs0Cnc', completion_window='24h', created_at=1725565297, endpoint='/v1/chat/completions', input_file_id='file-rbzekY6CP6dLdjCLi91jfcXD', object='batch', status='completed', cancelled_at=None, cancelling_at=None, completed_at=1725565635, error_file_id=None, errors=None, expired_at=None, expires_at=1725651697, failed_at=None, finalizing_at=1725565626, in_progress_at=1725565297, metadata={'description': 'Chat completions for file-rbzekY6CP6dLdjCLi91jfcXD'}, output_file_id='file-lSTh0Oz2Fw52X9BSs4XZz7qa', request_counts=BatchRequestCounts(completed=123, failed=0, total=123))\n",
166+
"Batch(id='batch_glNBmyi30uDTH0EuG3c6jTov', completion_window='24h', created_at=1725565295, endpoint='/v1/chat/completions', input_file_id='file-IHVfDSrWeL8YerJ6wMZvtuol', object='batch', status='completed', cancelled_at=None, cancelling_at=None, completed_at=1725567742, error_file_id=None, errors=None, expired_at=None, expires_at=1725651695, failed_at=None, finalizing_at=1725567075, in_progress_at=1725565298, metadata={'description': 'Chat completions for file-IHVfDSrWeL8YerJ6wMZvtuol'}, output_file_id='file-YPI6EQickl8rswaWJoJntLOO', request_counts=BatchRequestCounts(completed=5453, failed=0, total=5453))\n",
167+
"Batch(id='batch_Ufc5EkBcAbSceD4XzNF67KjO', completion_window='24h', created_at=1725549306, endpoint='/v1/chat/completions', input_file_id='file-PyayymAZ5TCcbTZUZ5nPN5zI', object='batch', status='completed', cancelled_at=None, cancelling_at=None, completed_at=1725549308, error_file_id=None, errors=None, expired_at=None, expires_at=1725635706, failed_at=None, finalizing_at=1725549308, in_progress_at=1725549306, metadata={'description': 'Batch Trial'}, output_file_id='file-O2iG9DpOYcU937Nrjru3JXxp', request_counts=BatchRequestCounts(completed=3, failed=0, total=3))\n"
168+
]
169+
}
170+
],
171+
"source": [
172+
"for _b in batches_info:\n",
173+
" print(_b)\n",
174+
"\n",
175+
" # if status is \"completed\" then download the file\n",
176+
" if _b.status != \"completed\":\n",
177+
" continue\n",
178+
"\n",
179+
" file_response = client.files.content(_b.output_file_id)\n",
180+
" with open(f\"../.scrapy/batch_prompts_results/{_b.id}.jsonl\", \"w\") as file:\n",
181+
" file.write(file_response.text)\n"
76182
]
77183
},
78184
{

‎micro-benchmark/python_features/classes/instance/main.py

-13
This file was deleted.

‎micro-benchmark/python_features/classes/instance/main_gt.json

-29
This file was deleted.

‎requirements.txt

+2-1
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,5 @@ PyYAML
99
requests==2.31.0
1010
tqdm
1111
tiktoken
12-
openai
12+
openai
13+
tabulate

‎scripts/benchmark_count.py

+11-2
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@
22
import os
33
from pathlib import Path
44

5-
microbench_folder_path = "../micro-benchmark/"
5+
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
6+
microbench_folder_path = f"{ROOT_DIR}/autogen_typeevalpy_benchmark"
67

78

89
def get_fact_stats(json_files):
@@ -68,6 +69,9 @@ def get_fact_stats(json_files):
6869
python_features_dir = microbench_folder_path + "/python_features"
6970
pf_overall_annotations = 0
7071
pf_overall_types = 0
72+
overall_functions = 0
73+
overall_params = 0
74+
overall_variables = 0
7175
for cat in sorted(os.listdir(python_features_dir)):
7276
cat_dir = os.path.join(python_features_dir, cat)
7377
json_files = [_file for _file in sorted(Path(cat_dir).rglob("*_gt.json"))]
@@ -76,10 +80,15 @@ def get_fact_stats(json_files):
7680

7781
pf_overall_annotations += _a
7882
pf_overall_types += _t
79-
83+
overall_functions += sum_functions
84+
overall_params += sum_params
85+
overall_variables += sum_variables
8086
print(cat, _a, _t, sum_functions, sum_params, sum_variables)
8187

8288
print(pf_overall_annotations, pf_overall_types)
89+
# print names
90+
print("overall_functions", "overall_params", "overall_variables")
91+
print(overall_functions, overall_params, overall_variables)
8392

8493
# analysis_sensitivities
8594
print("\nanalysis_sensitivities")

0 commit comments

Comments
 (0)