~drop litellm

Author: lnostdal

pyproject.toml

@@ -1,20 +1,21 @@
+# SPDX-License-Identifier: MPL-2.0 AND LicenseRef-Commons-Clause-License-Condition-1.0
+# <!-- // /*  d a r k s h a p e s */ -->
+
 [build-system]
-requires      = ["setuptools", "setuptools-scm"]
+requires = ["setuptools", "setuptools-scm"]
 build-backend = "setuptools.build_meta"
 
 [project]
-
 authors = [
     { name = "darkshapes", email = "91800957+exdysa@users.noreply.github.com" },
 ]
-description = "Multimodal generative media sequencer"
-license = { file = "LICENSE" }
 name = "zodiac"
+description = "Multimodal generative media sequencer"
 readme = "README.md"
-requires-python = ">=3.11"
 version = "0.0.2"
+license = { file = "LICENSE" }
+requires-python = ">=3.11"
 keywords = ["ML", "AI", "TUI", "no-code", "diffusion", "LLM", "torch"]
-
 classifiers = [
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
@@ -42,10 +43,10 @@ classifiers = [
     "Topic :: Multimedia :: Video :: Non-Linear Editor",
 
 ]
+
 dependencies = [
     "dspy>=2.6.27",
-    "litellm>=1.72.0",
-    "tokenizers==0.21.2",                                      #force for non linux
+    "tokenizers==0.21.2",                                            #force for non linux
     "networkx>=3.5",
     "sounddevice>=0.5.2",
     "soundfile>=0.13.1",
@@ -57,7 +58,7 @@ dependencies = [
     "pyperclip>=1.9.0",
     "ftfy>=6.3.1",
     "pdoc>=16.0.0",
-
+    "any-llm-sdk[ollama,openai]>=1.13.0",
 ]
 
 [dependency-groups]
@@ -72,7 +73,7 @@ dev = [
 
 
 [project.optional-dependencies]
-mps  = ["nnll[mps]@  git+https://github.com/darkshapes/nnll@0.1.dev799"]
+mps = ["nnll[mps]@  git+https://github.com/darkshapes/nnll@0.1.dev799"]
 cuda = ["nnll[cuda ]@ git+https://github.com/darkshapes/nnll@0.1.dev799"]
 
 console = [
@@ -95,13 +96,13 @@ dev = [
 ]
 
 [project.urls]
-Homepage      = "https://github.com/darkshapes/zodiac"
+Homepage = "https://github.com/darkshapes/zodiac"
 Documentation = "https://github.com/darkshapes/sdbx/wiki"
 
 [project.scripts]
-zdac      = "zodiac.toga.__main__:main"
+zdac = "zodiac.toga.__main__:main"
 zdac-pool = "zodiac.providers.pools:generate_pool"
-zodiac    = "zodiac.toga.__main__:main"
+zodiac = "zodiac.toga.__main__:main"
 
 [tool.setuptools]
 packages = ["zodiac"]
@@ -119,16 +120,16 @@ conflicts = [
     ],
 ]
 [tool.ruff]
-line-length    = 140
-include        = ["*.py"]
+line-length = 140
+include = ["*.py"]
 extend-exclude = ["^tests/.*$", "test.*$"]
 
 # [tool.ruff.format]
 # exclude = ["*.py"]
 
 [tool.pylint]
 max-line-length = 140
-ignore-paths    = ["^tests/.*$", "test_.*$"]
+ignore-paths = ["^tests/.*$", "test_.*$"]
 
 [tool.pylint.messages_control]
 disable = ["C0415"]
@@ -140,7 +141,7 @@ ignore = ["E731"]
 convention = "numpy"
 
 [tool.ruff.lint.pycodestyle]
-max-line-length               = 140
+max-line-length = 140
 ignore-overlong-task-comments = true
 
 [tool.typos]

uv.lock

@@ -48,23 +48,6 @@ def set_env(args: argparse.ArgumentParser):
         os.environ["HF_HUB_DISABLE_PROGRESS_BARS"] = str(not args.diag)
         os.environ["HF_HUB_OFFLINE"] = str(int(not args.net or not args.diag))
 
-        os.environ["DISABLE_HF_TOKENIZER_DOWNLOAD"] = str(not args.net or not args.diag)  # litellm
-        # huggingface_hub.constants.HF_HUB_VERBOSITY
-
-    try:
-        import litellm
-    except (ImportError, ModuleNotFoundError, Exception):  # pylint: disable=broad-exception-caught
-        pass
-    else:
-        litellm.disable_token_counter = False
-        litellm.disable_streaming_logging = True
-        litellm.turn_off_message_logging = True
-        litellm.suppress_debug_info = False
-        litellm.json_logs = False  # type: ignore
-        litellm.disable_end_user_cost_tracking = True
-        litellm.telemetry = False
-        litellm.disable_hf_tokenizer_download = not args.net  # -net = True -> disable download = False/0
-        os.environ["DISABLE_END_USER_COST_TRACKING"] = "True"
     return True
 
 

zodiac/__init__.py

@@ -3,18 +3,25 @@
 
 # pylint: disable=import-error
 
-
-import warnings
-from pathlib import Path
 from typing import Callable, Optional
 
-warnings.filterwarnings("ignore", category=DeprecationWarning)
-
-from litellm.utils import create_tokenizer, token_counter
+import tiktoken
 from toga.sources import Source
 from zodiac.providers.registry_entry import RegistryEntry
 
 
+async def tiktoken_counter(model="cl100k_base", message: str = ""):
+    """
+    Return token count of gpt based on model\n
+    :param model: Model path to lookup tokenizer for
+    :param message: Message to tokenize
+    :return: `int` Number of tokens needed to represent message
+    """
+
+    encoding = tiktoken.get_encoding(model)
+    return len(encoding.encode(message))
+
+
 class TokenStream(Source):
     def __init__(self):
         self.tokenizer: Optional[str] = None
@@ -27,18 +34,7 @@ async def set_tokenizer(self, registry_entry: RegistryEntry) -> Callable:
         :param message: Text to encode
         :return: Token embeddings for the model"""
 
-        import json
-
-        if registry_entry.tokenizer:
-            with open(str(registry_entry.tokenizer), encoding="UTF-8") as file_obj:
-                tokenizer_json = json.load(file_obj)
-                tokenizer_data = json.dumps(tokenizer_json)
-                self.tokenizer_args = {"custom_tokenizer": create_tokenizer(tokenizer_data)}
-        else:
-            # model_name = os.path.split(registry_entry.model)
-            # model_name = os.path.join(os.path.split(model_name[0])[-1], model_name[-1])
-            # self.status_log.registry_entry.model
-            self.tokenizer_args = {"model": registry_entry.model}
+        self.tokenizer_args = {}  # Disabled until suitable replacement is found
 
     async def token_count(
         self,
@@ -52,4 +48,4 @@ async def token_count(
 
         warnings.filterwarnings("ignore", category=DeprecationWarning)
         character_count = len(message)
-        return token_counter(text=message, **self.tokenizer_args), character_count
+        return tiktoken_counter(text=message, **self.tokenizer_args), character_count

zodiac/streams/token_stream.py

@@ -26,7 +26,7 @@
 class Interface(toga.App):
     formatted_units = [" ❖ chr", " ⟐ tok", ' " sec ']
     bg_graph = "#070708"
-    bg_text = "#1B1B1B"  # "#1B1B1B"  # "#09090B"
+    bg_text = "#1E1C1D"  # "#1B1B1B"  # "#09090B"
     bg = bg_text
     bg_static = "#5D5E62"
     activity = "#8122C4"
@@ -53,33 +53,29 @@ async def ticker(self, widget: Callable, external: bool = False, **kwargs) -> to
         if self.message_panel.value:
             cache = False
             prompts.setdefault("text", self.message_panel.value)
-        # prompts.setdefault("audio",[0]) if else []
-        # prompts.setdefault("image",[]) if image": []
         if stream := self.output_types.value == "text":
             context_data, predictor_data = await ready_predictor(self.registry_entry, dspy_stream=stream, async_stream=stream, cache=cache)
             await self.stream_text(prompts, context_data, predictor_data)
         else:
-            content = await self.generate_media(prompts, self.registry_entry)  # context_data, predictor_data)
             return widget
         return widget
 
     async def stream_text(self, prompts, context_data, predictor_data):
         from zodiac.toga.signatures import Predictor
-        from litellm.types.utils import ModelResponseStream  # StatusStreamingCallback
         from dspy.streaming import StatusMessage, StreamResponse
 
         self.response_panel.scroll_to_bottom()
         with dspy_context(**context_data):
             self.program = streamify(Predictor(), **predictor_data)
             async for prediction in self.program(question=prompts["text"]):
-                if isinstance(prediction, ModelResponseStream) and prediction["choices"][0]["delta"]["content"]:
-                    self.response_panel.value += prediction["choices"][0]["delta"]["content"]
-                elif isinstance(prediction, StreamResponse) or hasattr(prediction, "chunk"):
+                if isinstance(prediction, StreamResponse) or hasattr(prediction, "chunk"):
                     self.response_panel.value += str(prediction.chunk)
                 elif isinstance(prediction, Prediction) or hasattr(prediction, "answer"):
                     self.response_panel.value += str(prediction.answer)
                 elif isinstance(prediction, StatusMessage) or hasattr(prediction, "message"):
                     self.status_display.text = self.status_text_prefix + str(prediction.message)
+                elif prediction["choices"][0]["delta"].get("content"):
+                    self.response_panel.value += prediction["choices"][0]["delta"]["content"]
         self.response_panel.value += "\n--\n\n"
         return prediction
 

zodiac/toga/app.py

@@ -13,6 +13,7 @@ async def ticker(self, widget: Callable, external: bool = False, **kwargs) -> to
         :param external: Indicates whether the processing should be handled externally (e.g., via clipboard), defaults to False
         :type external: bool"""
         from zodiac.toga.signatures import ready_predictor
+        import os
 
         self.response_panel.value += f"{os.path.basename(self.registry_entry.model)} :\n"
 
@@ -34,21 +35,22 @@ async def ticker(self, widget: Callable, external: bool = False, **kwargs) -> to
 
     async def stream_text(self, prompts, context_data, predictor_data):
         from zodiac.toga.signatures import Predictor
-        from litellm.types.utils import ModelResponseStream  # StatusStreamingCallback
+
+        from dspy import Prediction, context as dspy_context, streamify
         from dspy.streaming import StatusMessage, StreamResponse
 
         self.response_panel.scroll_to_bottom()
         with dspy_context(**context_data):
             self.program = streamify(Predictor(), **predictor_data)
             async for prediction in self.program(question=prompts["text"]):
-                if isinstance(prediction, ModelResponseStream) and prediction["choices"][0]["delta"]["content"]:
-                    self.response_panel.value += prediction["choices"][0]["delta"]["content"]
-                elif isinstance(prediction, StreamResponse) or hasattr(prediction, "chunk"):
+                if isinstance(prediction, StreamResponse) or hasattr(prediction, "chunk"):
                     self.response_panel.value += str(prediction.chunk)
                 elif isinstance(prediction, Prediction) or hasattr(prediction, "answer"):
                     self.response_panel.value += str(prediction.answer)
                 elif isinstance(prediction, StatusMessage) or hasattr(prediction, "message"):
                     self.status_display.text = self.status_text_prefix + str(prediction.message)
+                elif prediction["choices"][0]["delta"].get("content"):
+                    self.response_panel.value += prediction["choices"][0]["delta"]["content"]
         self.response_panel.value += "\n--\n\n"
         return prediction
 

zodiac/toga/palette.py

@@ -87,15 +87,14 @@ def __init__(self):
         self.program = dspy.Predict(signature=QATask)
 
     def __call__(self, question: str):
-        from litellm.exceptions import APIConnectionError
-        from litellm.llms.ollama.common_utils import OllamaError
+        from any_llm.utils.exception_handler import AnyLLMError
         from httpx import ConnectError
         from dspy.utils.exceptions import AdapterParseError
         from aiohttp.client_exceptions import ClientConnectorError
 
         try:
             return self.program(question=question)
-        except (ClientConnectorError, ConnectError, AdapterParseError, APIConnectionError, OllamaError, OSError):
+        except (ClientConnectorError, ConnectError, AdapterParseError, AnyLLMError, OSError):
             pass
 
     # from nnll.tensor_pipe import segments