LLMs | Training set and jsonl for fine tuning

Author: Samzcodez

src/target_tools/ollama/fine_tuning/create_jsonl.py

@@ -0,0 +1,60 @@
+import os
+import json
+
+
+def process_file(file_path):
+    with open(file_path, "r") as f:
+        return f.read()
+
+
+def process_json_data(json_data):
+    return json.loads(json_data)
+
+
+def process_folder(folder_path, output_file):
+    with open(output_file, "w") as output:
+        for root, dirs, files in os.walk(folder_path):
+            messages = []
+
+            system_message = {
+                "role": "system",
+                "content": (
+                    "Perform the following tasks: 1. Infer the types of various Python"
+                    " elements like function parameters, local variables, and function"
+                    " return types with the highest probability. 2. Provide your"
+                    " response in a valid JSON array of objects. Do not provide any"
+                    " additional information except the JSON object."
+                ),
+            }
+            messages.append(system_message)
+            print("Analysing:", root)
+            for file_name in files:
+                file_path = os.path.join(root, file_name)
+
+                if file_name.endswith(".py"):
+                    code = process_file(file_path)
+                    user_message = {
+                        "role": "user",
+                        "content": f"'{file_name}':\n\n```\n{code}\n```",
+                    }
+                    messages.append(user_message)
+
+                elif file_name.endswith(".json"):
+                    assistant_content = process_file(file_path)
+                    data = process_json_data(assistant_content)
+                    assistant_message = {
+                        "role": "assistant",
+                        "content": json.dumps(data),
+                    }
+                    messages.append(assistant_message)
+
+            # Write messages to the output file
+            if any(entry.get("role") == "user" for entry in messages):
+                output.write(
+                    json.dumps({"messages": messages}, separators=(",", ":")) + "\n"
+                )
+
+
+folder_path = "training_set"
+output_file = "output.jsonl"
+process_folder(folder_path, output_file)

src/target_tools/ollama/fine_tuning/output.jsonl

@@ -0,0 +1,11 @@
+{"messages":[{"role":"system","content":"Perform the following tasks: 1. Infer the types of various Python elements like function parameters, local variables, and function return types with the highest probability. 2. Provide your response in a valid JSON array of objects. Do not provide any additional information except the JSON object."},{"role":"user","content":"'main.py':\n\n```\ndef func():\n    return \"Hello from func\"\n\n```"},{"role":"assistant","content":"[{\"file\": \"main.py\", \"line_number\": 1, \"function\": \"func\", \"type\": [\"str\"]}]"}]}
+{"messages":[{"role":"system","content":"Perform the following tasks: 1. Infer the types of various Python elements like function parameters, local variables, and function return types with the highest probability. 2. Provide your response in a valid JSON array of objects. Do not provide any additional information except the JSON object."},{"role":"user","content":"'main.py':\n\n```\ndef func(a):\n    return \"Hello from func\"\n\n\nresult = func(5)\n\n```"},{"role":"assistant","content":"[{\"file\": \"main.py\", \"line_number\": 1, \"function\": \"func\", \"type\": [\"str\"]}, {\"file\": \"main.py\", \"line_number\": 1, \"function\": \"func\", \"parameter\": \"a\", \"type\": [\"int\"]}, {\"file\": \"main.py\", \"line_number\": 5, \"variable\": \"result\", \"type\": [\"str\"]}]"}]}
+{"messages":[{"role":"system","content":"Perform the following tasks: 1. Infer the types of various Python elements like function parameters, local variables, and function return types with the highest probability. 2. Provide your response in a valid JSON array of objects. Do not provide any additional information except the JSON object."},{"role":"user","content":"'main.py':\n\n```\ndef func():\n    x = 1\n    return x\n\n\nresult1 = func()\n\n```"},{"role":"assistant","content":"[{\"file\": \"main.py\", \"line_number\": 1, \"function\": \"func\", \"type\": [\"int\"]}, {\"file\": \"main.py\", \"line_number\": 1, \"function\": \"func\", \"variable\": \"x\", \"type\": [\"int\"]}, {\"file\": \"main.py\", \"line_number\": 6, \"variable\": \"result1\", \"type\": [\"int\"]}]"}]}
+{"messages":[{"role":"system","content":"Perform the following tasks: 1. Infer the types of various Python elements like function parameters, local variables, and function return types with the highest probability. 2. Provide your response in a valid JSON array of objects. Do not provide any additional information except the JSON object."},{"role":"user","content":"'main.py':\n\n```\na = 10\n\n```"},{"role":"assistant","content":"[{\"file\": \"main.py\", \"line_number\": 1, \"variable\": \"a\", \"type\": [\"int\"]}]"}]}
+{"messages":[{"role":"system","content":"Perform the following tasks: 1. Infer the types of various Python elements like function parameters, local variables, and function return types with the highest probability. 2. Provide your response in a valid JSON array of objects. Do not provide any additional information except the JSON object."},{"role":"user","content":"'main.py':\n\n```\ndef outer():\n    x = 1\n\n    def inner():\n        y = \"Hello\"\n        return y\n\n    return inner()\n\n\nresult1 = outer()\n\n```"},{"role":"assistant","content":"[{\"file\": \"main.py\", \"line_number\": 1, \"function\": \"outer\", \"type\": [\"str\"]}, {\"file\": \"main.py\", \"line_number\": 2, \"function\": \"outer\", \"variable\": \"x\", \"type\": [\"int\"]}, {\"file\": \"main.py\", \"line_number\": 4, \"function\": \"inner\", \"type\": [\"str\"]}, {\"file\": \"main.py\", \"line_number\": 5, \"function\": \"inner\", \"variable\": \"y\", \"type\": [\"str\"]}, {\"file\": \"main.py\", \"line_number\": 11, \"variable\": \"result1\", \"type\": [\"str\"]}]"}]}
+{"messages":[{"role":"system","content":"Perform the following tasks: 1. Infer the types of various Python elements like function parameters, local variables, and function return types with the highest probability. 2. Provide your response in a valid JSON array of objects. Do not provide any additional information except the JSON object."},{"role":"user","content":"'main.py':\n\n```\na = [3, 5, 2]\n\n```"},{"role":"assistant","content":"[{\"file\": \" main.py \", \"line_number\": 1, \"variable\": \"a\", \"type\": [\"list\"]}]"}]}
+{"messages":[{"role":"system","content":"Perform the following tasks: 1. Infer the types of various Python elements like function parameters, local variables, and function return types with the highest probability. 2. Provide your response in a valid JSON array of objects. Do not provide any additional information except the JSON object."},{"role":"user","content":"'main.py':\n\n```\ndef func(a=5):\n    return\n\n```"},{"role":"assistant","content":"[{\"file\": \"main.py\", \"line_number\": 1, \"function\": \"func\", \"type\": [\"Nonetype\"]}, {\"file\": \"main.py\", \"line_number\": 1, \"function\": \"func\", \"parameter\": \"a\", \"type\": [\"int\"]}]"}]}
+{"messages":[{"role":"system","content":"Perform the following tasks: 1. Infer the types of various Python elements like function parameters, local variables, and function return types with the highest probability. 2. Provide your response in a valid JSON array of objects. Do not provide any additional information except the JSON object."},{"role":"user","content":"'main.py':\n\n```\ndef add(a, b):\n    return a + b\n\n\nresult1 = add(2, 3)\nresult2 = add(\"Hello\", \"String\")\n\n```"},{"role":"assistant","content":"[{\"file\": \"main.py\", \"line_number\": 1, \"function\": \"add\", \"type\": [\"int\", \"str\"]}, {\"file\": \"main.py\", \"line_number\": 1, \"function\": \"func\", \"parameter\": \"a\", \"type\": [\"int\", \"str\"]}, {\"file\": \"main.py\", \"line_number\": 1, \"function\": \"func\", \"parameter\": \"b\", \"type\": [\"int\", \"str\"]}, {\"file\": \"main.py\", \"line_number\": 5, \"variable\": \"result1\", \"type\": [\"int\"]}, {\"file\": \"main.py\", \"line_number\": 6, \"variable\": \"result2\", \"type\": [\"str\"]}]"}]}
+{"messages":[{"role":"system","content":"Perform the following tasks: 1. Infer the types of various Python elements like function parameters, local variables, and function return types with the highest probability. 2. Provide your response in a valid JSON array of objects. Do not provide any additional information except the JSON object."},{"role":"user","content":"'main.py':\n\n```\ndict1 = {\"a\": 1, \"b\": 2}\n\n```"},{"role":"assistant","content":"[{\"file\": \"main.py\", \"line_number\": 4, \"col_offset\": 1, \"variable\": \"dict1\", \"type\": [\"dict\"]}, {\"file\": \"main.py\", \"line_number\": 4, \"col_offset\": 1, \"variable\": \"dict1['a']\", \"type\": [\"int\"]}, {\"file\": \"main.py\", \"line_number\": 4, \"col_offset\": 1, \"variable\": \"dict1['b']\", \"type\": [\"int\"]}]"}]}
+{"messages":[{"role":"system","content":"Perform the following tasks: 1. Infer the types of various Python elements like function parameters, local variables, and function return types with the highest probability. 2. Provide your response in a valid JSON array of objects. Do not provide any additional information except the JSON object."},{"role":"user","content":"'main.py':\n\n```\ndef func():\n    return \"Hello from func\"\n\n\na = func\n\n```"},{"role":"assistant","content":"[{\"file\": \"main.py\", \"line_number\": 1, \"function\": \"func\", \"type\": [\"str\"]}, {\"file\": \"main.py\", \"line_number\": 5, \"variable\": \"a\", \"type\": [\"callable\"]}]"}]}
+{"messages":[{"role":"system","content":"Perform the following tasks: 1. Infer the types of various Python elements like function parameters, local variables, and function return types with the highest probability. 2. Provide your response in a valid JSON array of objects. Do not provide any additional information except the JSON object."},{"role":"user","content":"'main.py':\n\n```\nclass TestClass:\n    def func(self):\n        return \"Hello from func in TestClass\"\n\n\ntest = TestClass()\nresult = test.func()\n\n```"},{"role":"assistant","content":"[{\"file\": \" main.py \", \"line_number\": 1, \"function\": \"TestClass.func\", \"type\": [\"str\"]}, {\"file\": \" main.py \", \"line_number\": 6, \"variable\": \"test\", \"type\": [\"TestClass\"]}, {\"file\": \" main.py \", \"line_number\": 7, \"variable\": \"result\", \"type\": [\"str\"]}]"}]}

src/target_tools/ollama/fine_tuning/training_set/1.Local_variable/main.py

@@ -0,0 +1 @@
+a = 10

src/target_tools/ollama/fine_tuning/training_set/1.Local_variable/main_gt.json

@@ -0,0 +1,10 @@
+[
+    {
+        "file": "main.py",
+        "line_number": 1,
+        "variable": "a",
+        "type": [
+            "int"
+        ]
+    }
+]

src/target_tools/ollama/fine_tuning/training_set/10.Callable_type/main.py

@@ -0,0 +1,5 @@
+def func():
+    return "Hello from func"
+
+
+a = func

src/target_tools/ollama/fine_tuning/training_set/10.Callable_type/main_gt.json

@@ -0,0 +1,18 @@
+[
+    {
+        "file": "main.py",
+        "line_number": 1,
+        "function": "func",
+        "type": [
+            "str"
+        ]
+    },
+    {
+        "file": "main.py",
+        "line_number": 5,
+        "variable": "a",
+        "type": [
+            "callable"
+        ]
+    }
+]

src/target_tools/ollama/fine_tuning/training_set/11.Nested_function/main.py

@@ -0,0 +1,11 @@
+def outer():
+    x = 1
+
+    def inner():
+        y = "Hello"
+        return y
+
+    return inner()
+
+
+result1 = outer()

src/target_tools/ollama/fine_tuning/training_set/11.Nested_function/main_gt.json

@@ -0,0 +1,44 @@
+[
+    {
+        "file": "main.py",
+        "line_number": 1,
+        "function": "outer",
+        "type": [
+            "str"
+        ]
+    },
+    {
+        "file": "main.py",
+        "line_number": 2,
+        "function": "outer",
+        "variable": "x",
+        "type": [
+            "int"
+        ]
+    },
+    {
+        "file": "main.py",
+        "line_number": 4,
+        "function": "inner",
+        "type": [
+            "str"
+        ]
+    },
+    {
+        "file": "main.py",
+        "line_number": 5,
+        "function": "inner",
+        "variable": "y",
+        "type": [
+            "str"
+        ]
+    },
+    {
+        "file": "main.py",
+        "line_number": 11,
+        "variable": "result1",
+        "type": [
+            "str"
+        ]
+    }
+]

src/target_tools/ollama/fine_tuning/training_set/2.Function_return/main.py

@@ -0,0 +1,2 @@
+def func():
+    return "Hello from func"

src/target_tools/ollama/fine_tuning/training_set/2.Function_return/main_gt.json

@@ -0,0 +1,10 @@
+[
+    {
+        "file": "main.py",
+        "line_number": 1,
+        "function": "func",
+        "type": [
+            "str"
+        ]
+    }
+]

src/target_tools/ollama/fine_tuning/training_set/3.Function_parameter/main.py

@@ -0,0 +1,2 @@
+def func(a=5):
+    return

src/target_tools/ollama/fine_tuning/training_set/3.Function_parameter/main_gt.json

@@ -0,0 +1,19 @@
+[
+    {
+        "file": "main.py",
+        "line_number": 1,
+        "function": "func",
+        "type": [
+            "Nonetype"
+        ]
+    },
+    {
+        "file": "main.py",
+        "line_number": 1,
+        "function": "func",
+        "parameter": "a",
+        "type": [
+            "int"
+        ]
+    }
+]

src/target_tools/ollama/fine_tuning/training_set/4.All_categories/main.py

@@ -0,0 +1,5 @@
+def func(a):
+    return "Hello from func"
+
+
+result = func(5)

src/target_tools/ollama/fine_tuning/training_set/4.All_categories/main_gt.json

@@ -0,0 +1,27 @@
+[
+    {
+        "file": "main.py",
+        "line_number": 1,
+        "function": "func",
+        "type": [
+            "str"
+        ]
+    },
+    {
+        "file": "main.py",
+        "line_number": 1,
+        "function": "func",
+        "parameter": "a",
+        "type": [
+            "int"
+        ]
+    },
+    {
+        "file": "main.py",
+        "line_number": 5,
+        "variable": "result",
+        "type": [
+            "str"
+        ]
+    }
+]

src/target_tools/ollama/fine_tuning/training_set/5.Function_variable/main.py

@@ -0,0 +1,6 @@
+def func():
+    x = 1
+    return x
+
+
+result1 = func()

src/target_tools/ollama/fine_tuning/training_set/5.Function_variable/main_gt.json

@@ -0,0 +1,27 @@
+[
+    {
+        "file": "main.py",
+        "line_number": 1,
+        "function": "func",
+        "type": [
+            "int"
+        ]
+    },
+    {
+        "file": "main.py",
+        "line_number": 1,
+        "function": "func",
+        "variable": "x",
+        "type": [
+            "int"
+        ]
+    },
+    {
+        "file": "main.py",
+        "line_number": 6,
+        "variable": "result1",
+        "type": [
+            "int"
+        ]
+    }
+]

src/target_tools/ollama/fine_tuning/training_set/6.Multiple_types/main.py

@@ -0,0 +1,6 @@
+def add(a, b):
+    return a + b
+
+
+result1 = add(2, 3)
+result2 = add("Hello", "String")

src/target_tools/ollama/fine_tuning/training_set/6.Multiple_types/main_gt.json

@@ -0,0 +1,47 @@
+[
+    {
+        "file": "main.py",
+        "line_number": 1,
+        "function": "add",
+        "type": [
+            "int",
+            "str"
+        ]
+    },
+    {
+        "file": "main.py",
+        "line_number": 1,
+        "function": "func",
+        "parameter": "a",
+        "type": [
+            "int",
+            "str"
+        ]
+    },
+    {
+        "file": "main.py",
+        "line_number": 1,
+        "function": "func",
+        "parameter": "b",
+        "type": [
+            "int",
+            "str"
+        ]
+    },
+    {
+        "file": "main.py",
+        "line_number": 5,
+        "variable": "result1",
+        "type": [
+            "int"
+        ]
+    },
+    {
+        "file": "main.py",
+        "line_number": 6,
+        "variable": "result2",
+        "type": [
+            "str"
+        ]
+    }
+]

src/target_tools/ollama/fine_tuning/training_set/7.Class/main.py

@@ -0,0 +1,7 @@
+class TestClass:
+    def func(self):
+        return "Hello from func in TestClass"
+
+
+test = TestClass()
+result = test.func()

src/target_tools/ollama/fine_tuning/training_set/7.Class/main_gt.json

@@ -0,0 +1,26 @@
+[
+    {
+        "file": " main.py ",
+        "line_number": 1,
+        "function": "TestClass.func",
+        "type": [
+            "str"
+        ]
+    },
+    {
+        "file": " main.py ",
+        "line_number": 6,
+        "variable": "test",
+        "type": [
+            "TestClass"
+        ]
+    },
+    {
+        "file": " main.py ",
+        "line_number": 7,
+        "variable": "result",
+        "type": [
+            "str"
+        ]
+    }
+]

src/target_tools/ollama/fine_tuning/training_set/8.List_basetype/main.py

@@ -0,0 +1 @@
+a = [3, 5, 2]

src/target_tools/ollama/fine_tuning/training_set/8.List_basetype/main_gt.json

@@ -0,0 +1,10 @@
+[
+    {
+        "file": " main.py ",
+        "line_number": 1,
+        "variable": "a",
+        "type": [
+            "list"
+        ]
+    }
+]

src/target_tools/ollama/fine_tuning/training_set/9.Dicts_index/main.py

@@ -0,0 +1 @@
+dict1 = {"a": 1, "b": 2}