improve: [llm_adapter] fix support for ollama deepseek-v2 which doesn…

…ot support function_call
luochen1990 · Jul 31, 2024 · 21ef687 · 21ef687
1 parent 37800b9
commit 21ef687
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 8 deletions.
diff --git a/src/ai_powered/llm_adapter/generic_adapter.py b/src/ai_powered/llm_adapter/generic_adapter.py
@@ -50,8 +50,8 @@ def query_model(self, user_msg: str) -> str:
         response = client.chat.completions.create(
             model = self.model_name,
             messages = messages,
-            tools = tools,
-            tool_choice = {"type": "function", "function": {"name": "return_result"}},
+            tools = tools if "function_call" in self.model_features else openai.NOT_GIVEN,
+            tool_choice = {"type": "function", "function": {"name": "return_result"}} if "function_call" in self.model_features else openai.NOT_GIVEN,
         )
         if DEBUG:
             print(yellow(f"{response =}"))
@@ -66,17 +66,18 @@ def query_model(self, user_msg: str) -> str:
             raw_resp_str = resp_msg.content
             assert raw_resp_str is not None
 
+            raw_resp_str_strip = raw_resp_str.strip()
             # raw_resp_str = "```json\n{"result": 2}\n```"
 
             is_markdown : Callable[[str], bool]= lambda s: s.startswith("```") and s.endswith("```")
 
-            if is_markdown(raw_resp_str):
-                if raw_resp_str.startswith("```json"):
-                    unwrapped_resp_str = raw_resp_str[7:-3]
+            if is_markdown(raw_resp_str_strip):
+                if raw_resp_str_strip.startswith("```json"):
+                    unwrapped_resp_str = raw_resp_str_strip[7:-3]
                 else:
-                    unwrapped_resp_str = raw_resp_str[3:-3]
+                    unwrapped_resp_str = raw_resp_str_strip[3:-3]
             else:
-                unwrapped_resp_str = raw_resp_str
+                unwrapped_resp_str = raw_resp_str_strip
 
             if DEBUG:
                 print(f"{unwrapped_resp_str =}")
@@ -91,7 +92,7 @@ def query_model(self, user_msg: str) -> str:
 
             if DEBUG:
                 print(f"{raw_resp_str =}")
-                print(f"{is_markdown(raw_resp_str) =}")
+                print(f"{is_markdown(raw_resp_str_strip) =}")
                 print(f"{is_result(unwrapped_resp_str) =}")
                 print(f"{result_str =}")
 

diff --git a/src/ai_powered/llm_adapter/known_models.py b/src/ai_powered/llm_adapter/known_models.py
@@ -56,6 +56,7 @@ def equals(s: str) -> Callable[[str], bool]:
         known_model_list = [
             KnownModel("gorilla-llm/gorilla-openfunctions-v2-gguf/gorilla-openfunctions-v2-q4_K_M.gguf", ALL_FEATURES),
             KnownModel("lmstudio-community/Meta-Llama-3-8B-Instruct-GGUF/Meta-Llama-3-8B-Instruct-Q4_K_M.gguf", set()),
+            KnownModel("deepseek-coder-v2", set()),
         ]
     ),
 ]