new: replace union with | in token count

Author: joein

fastembed/late_interaction/colbert.py

@@ -98,7 +98,7 @@ def _tokenize_documents(self, documents: list[str]) -> list[Encoding]:
 
     def token_count(
         self,
-        texts: Union[str, Iterable[str]],
+        texts: str | Iterable[str],
         batch_size: int = 1024,
         is_doc: bool = True,
         include_extension: bool = False,

fastembed/late_interaction/late_interaction_embedding_base.py

@@ -72,7 +72,7 @@ def embedding_size(self) -> int:
 
     def token_count(
         self,
-        texts: Union[str, Iterable[str]],
+        texts: str | Iterable[str],
         batch_size: int = 1024,
         **kwargs: Any,
     ) -> int:

fastembed/late_interaction/late_interaction_text_embedding.py

@@ -154,7 +154,7 @@ def query_embed(self, query: str | Iterable[str], **kwargs: Any) -> Iterable[Num
 
     def token_count(
         self,
-        texts: Union[str, Iterable[str]],
+        texts: str | Iterable[str],
         batch_size: int = 1024,
         is_doc: bool = True,
         include_extension: bool = False,

fastembed/late_interaction_multimodal/colpali.py

@@ -174,7 +174,7 @@ def tokenize(self, documents: list[str], **kwargs: Any) -> list[Encoding]:
 
     def token_count(
         self,
-        texts: Union[str, Iterable[str]],
+        texts: str | Iterable[str],
         batch_size: int = 1024,
         include_extension: bool = False,
         **kwargs: Any,

fastembed/late_interaction_multimodal/late_interaction_multimodal_embedding.py

@@ -165,7 +165,7 @@ def embed_image(
 
     def token_count(
         self,
-        texts: Union[str, Iterable[str]],
+        texts: str | Iterable[str],
         batch_size: int = 1024,
         include_extension: bool = False,
         **kwargs: Any,

fastembed/late_interaction_multimodal/late_interaction_multimodal_embedding_base.py

@@ -79,7 +79,7 @@ def embedding_size(self) -> int:
 
     def token_count(
         self,
-        texts: Union[str, Iterable[str]],
+        texts: str | Iterable[str],
         **kwargs: Any,
     ) -> int:
         """Returns the number of tokens in the texts."""

fastembed/sparse/bm25.py

@@ -268,7 +268,7 @@ def raw_embed(
             embeddings.append(SparseEmbedding.from_dict(token_id2value))
         return embeddings
 
-    def token_count(self, texts: Union[str, Iterable[str]], **kwargs: Any) -> int:
+    def token_count(self, texts: str | Iterable[str], **kwargs: Any) -> int:
         token_num = 0
         texts = [texts] if isinstance(texts, str) else texts
         for text in texts:

fastembed/sparse/bm42.py

@@ -351,7 +351,7 @@ def _get_worker_class(cls) -> Type[TextEmbeddingWorker[SparseEmbedding]]:
         return Bm42TextEmbeddingWorker
 
     def token_count(
-        self, texts: Union[str, Iterable[str]], batch_size: int = 1024, **kwargs: Any
+        self, texts: str | Iterable[str], batch_size: int = 1024, **kwargs: Any
     ) -> int:
         if not hasattr(self, "model") or self.model is None:
             self.load_onnx_model()  # loads the tokenizer as well

fastembed/sparse/minicoil.py

@@ -188,7 +188,7 @@ def load_onnx_model(self) -> None:
         )
 
     def token_count(
-        self, texts: Union[str, Iterable[str]], batch_size: int = 1024, **kwargs: Any
+        self, texts: str | Iterable[str], batch_size: int = 1024, **kwargs: Any
     ) -> int:
         return self._token_count(texts, batch_size=batch_size, **kwargs)
 

fastembed/sparse/sparse_embedding_base.py

@@ -85,6 +85,6 @@ def query_embed(self, query: str | Iterable[str], **kwargs: Any) -> Iterable[Spa
         else:
             yield from self.embed(query, **kwargs)
 
-    def token_count(self, texts: Union[str, Iterable[str]], **kwargs: Any) -> int:
+    def token_count(self, texts: str | Iterable[str], **kwargs: Any) -> int:
         """Returns the number of tokens in the texts."""
         raise NotImplementedError("Subclasses must implement this method")