feat: New open source models (#649)

Co-authored-by: Wendong-Fan <[email protected]>
Co-authored-by: Wendong <[email protected]>

Author: zhiyu-01

camel/types/enums.py

@@ -30,15 +30,19 @@ class ModelType(Enum):
     GPT_4_TURBO = "gpt-4-turbo"
     GPT_4O = "gpt-4o"
     GLM_4 = "glm-4"
+    GLM_4_OPEN_SOURCE = "glm-4-open-source"
     GLM_4V = 'glm-4v'
     GLM_3_TURBO = "glm-3-turbo"
 
     STUB = "stub"
 
     LLAMA_2 = "llama-2"
+    LLAMA_3 = "llama-3"
     VICUNA = "vicuna"
     VICUNA_16K = "vicuna-16k"
 
+    QWEN_2 = "qwen-2"
+
     # Legacy anthropic models
     # NOTE: anthropic lagecy models only Claude 2.1 has system prompt support
     CLAUDE_2_1 = "claude-2.1"
@@ -87,6 +91,9 @@ def is_open_source(self) -> bool:
         r"""Returns whether this type of models is open-source."""
         return self in {
             ModelType.LLAMA_2,
+            ModelType.LLAMA_3,
+            ModelType.QWEN_2,
+            ModelType.GLM_4_OPEN_SOURCE,
             ModelType.VICUNA,
             ModelType.VICUNA_16K,
         }
@@ -135,7 +142,7 @@ def token_limit(self) -> int:
             return 128000
         elif self is ModelType.GPT_4O:
             return 128000
-        elif self == ModelType.GLM_4:
+        elif self == ModelType.GLM_4_OPEN_SOURCE:
             return 8192
         elif self == ModelType.GLM_3_TURBO:
             return 8192
@@ -145,6 +152,12 @@ def token_limit(self) -> int:
             return 4096
         elif self is ModelType.LLAMA_2:
             return 4096
+        elif self is ModelType.LLAMA_3:
+            return 8192
+        elif self is ModelType.QWEN_2:
+            return 128000
+        elif self is ModelType.GLM_4:
+            return 8192
         elif self is ModelType.VICUNA:
             # reference: https://lmsys.org/blog/2023-03-30-vicuna/
             return 2048
@@ -184,6 +197,20 @@ def validate_model_name(self, model_name: str) -> bool:
                 self.value in model_name.lower()
                 or "llama2" in model_name.lower()
             )
+        elif self is ModelType.LLAMA_3:
+            return (
+                self.value in model_name.lower()
+                or "llama3" in model_name.lower()
+            )
+        elif self is ModelType.QWEN_2:
+            return (
+                self.value in model_name.lower()
+                or "qwen2" in model_name.lower()
+            )
+        elif self is ModelType.GLM_4_OPEN_SOURCE:
+            return (
+                'glm-4' in model_name.lower() or "glm4" in model_name.lower()
+            )
         else:
             return self.value in model_name.lower()
 

camel/utils/token_counting.py

@@ -51,7 +51,7 @@ def messages_to_prompt(messages: List[OpenAIMessage], model: ModelType) -> str:
     system_message = messages[0]["content"]
 
     ret: str
-    if model == ModelType.LLAMA_2:
+    if model == ModelType.LLAMA_2 or model == ModelType.LLAMA_3:
         # reference: https://github.com/facebookresearch/llama/blob/cfc3fc8c1968d390eb830e65c63865e980873a06/llama/generation.py#L212
         seps = [" ", " </s><s>"]
         role_map = {"user": "[INST]", "assistant": "[/INST]"}
@@ -93,6 +93,45 @@ def messages_to_prompt(messages: List[OpenAIMessage], model: ModelType) -> str:
             else:
                 ret += role + ":"
         return ret
+    elif model == ModelType.GLM_4_OPEN_SOURCE:
+        system_prompt = f"[gMASK]<sop><|system|>\n{system_message}"
+        ret = system_prompt
+        for msg in messages[1:]:
+            role = msg["role"]
+            content = msg["content"]
+            if not isinstance(content, str):
+                raise ValueError(
+                    "Currently multimodal context is not "
+                    "supported by the token counter."
+                )
+            if content:
+                ret += "<|" + role + "|>" + "\n" + content
+            else:
+                ret += "<|" + role + "|>" + "\n"
+        return ret
+    elif model == ModelType.QWEN_2:
+        system_prompt = f"<|im_start|>system\n{system_message}<|im_end|>"
+        ret = system_prompt + "\n"
+        for msg in messages[1:]:
+            role = msg["role"]
+            content = msg["content"]
+            if not isinstance(content, str):
+                raise ValueError(
+                    "Currently multimodal context is not "
+                    "supported by the token counter."
+                )
+            if content:
+                ret += (
+                    '<|im_start|>'
+                    + role
+                    + '\n'
+                    + content
+                    + '<|im_end|>'
+                    + '\n'
+                )
+            else:
+                ret += '<|im_start|>' + role + '\n'
+        return ret
     else:
         raise ValueError(f"Invalid model type: {model}")