|
| 1 | +from typing import Any, Dict, Iterable, List, Optional, Union, Type, Sequence |
| 2 | +import string |
| 3 | + |
| 4 | +import numpy as np |
| 5 | +from tokenizers import Encoding |
| 6 | + |
| 7 | +from fastembed.common import OnnxProvider |
| 8 | +from fastembed.common.onnx_model import OnnxOutputContext |
| 9 | +from fastembed.common.utils import define_cache_dir |
| 10 | +from fastembed.late_interaction.late_interaction_embedding_base import ( |
| 11 | + LateInteractionTextEmbeddingBase, |
| 12 | +) |
| 13 | +from fastembed.text.onnx_text_model import OnnxTextModel, TextEmbeddingWorker |
| 14 | + |
| 15 | + |
| 16 | +supported_colbert_models = [ |
| 17 | + { |
| 18 | + "model": "colbert-ir/colbertv2.0", |
| 19 | + "dim": 128, |
| 20 | + "description": "Late interaction model", |
| 21 | + "size_in_GB": 0.44, |
| 22 | + "sources": { |
| 23 | + "hf": "colbert-ir/colbertv2.0", |
| 24 | + }, |
| 25 | + "model_file": "model.onnx", |
| 26 | + } |
| 27 | +] |
| 28 | + |
| 29 | + |
| 30 | +class Colbert(LateInteractionTextEmbeddingBase, OnnxTextModel[np.ndarray]): |
| 31 | + QUERY_MARKER_TOKEN_ID = 1 |
| 32 | + DOCUMENT_MARKER_TOKEN_ID = 2 |
| 33 | + MIN_QUERY_LENGTH = 32 |
| 34 | + MASK_TOKEN = "[MASK]" |
| 35 | + |
| 36 | + def _post_process_onnx_output( |
| 37 | + self, output: OnnxOutputContext, is_doc: bool = True |
| 38 | + ) -> Iterable[np.ndarray]: |
| 39 | + if not is_doc: |
| 40 | + return output.model_output.astype(np.float32) |
| 41 | + |
| 42 | + for i, token_sequence in enumerate(output.input_ids): |
| 43 | + for j, token_id in enumerate(token_sequence): |
| 44 | + if token_id in self.skip_list or token_id == self.pad_token_id: |
| 45 | + output.attention_mask[i, j] = 0 |
| 46 | + |
| 47 | + output.model_output *= np.expand_dims(output.attention_mask, 2).astype(np.float32) |
| 48 | + norm = np.linalg.norm(output.model_output, ord=2, axis=2, keepdims=True) |
| 49 | + norm_clamped = np.maximum(norm, 1e-12) |
| 50 | + output.model_output /= norm_clamped |
| 51 | + return output.model_output.astype(np.float32) |
| 52 | + |
| 53 | + def _preprocess_onnx_input( |
| 54 | + self, onnx_input: Dict[str, np.ndarray], is_doc: bool = True |
| 55 | + ) -> Dict[str, np.ndarray]: |
| 56 | + if is_doc: |
| 57 | + onnx_input["input_ids"][:, 1] = self.DOCUMENT_MARKER_TOKEN_ID |
| 58 | + else: |
| 59 | + onnx_input["input_ids"][:, 1] = self.QUERY_MARKER_TOKEN_ID |
| 60 | + return onnx_input |
| 61 | + |
| 62 | + def tokenize(self, documents: List[str], is_doc: bool = True) -> List[Encoding]: |
| 63 | + return ( |
| 64 | + self._tokenize_documents(documents=documents) |
| 65 | + if is_doc |
| 66 | + else self._tokenize_query(query=next(iter(documents))) |
| 67 | + ) |
| 68 | + |
| 69 | + def _tokenize_query(self, query: str) -> List[Encoding]: |
| 70 | + # ". " is added to a query to be replaced with a special query token |
| 71 | + query = [f". {query}"] |
| 72 | + encoded = self.tokenizer.encode_batch(query) |
| 73 | + # colbert authors recommend to pad queries with [MASK] tokens for query augmentation to improve performance |
| 74 | + if len(encoded[0].ids) < self.MIN_QUERY_LENGTH: |
| 75 | + prev_padding = None |
| 76 | + if self.tokenizer.padding: |
| 77 | + prev_padding = self.tokenizer.padding |
| 78 | + self.tokenizer.enable_padding( |
| 79 | + pad_token=self.MASK_TOKEN, pad_id=self.mask_token_id, length=self.MIN_QUERY_LENGTH |
| 80 | + ) |
| 81 | + encoded = self.tokenizer.encode_batch(query) |
| 82 | + if prev_padding is None: |
| 83 | + self.tokenizer.no_padding() |
| 84 | + else: |
| 85 | + self.tokenizer.enable_padding(**prev_padding) |
| 86 | + return encoded |
| 87 | + |
| 88 | + def _tokenize_documents(self, documents: List[str]) -> List[Encoding]: |
| 89 | + # ". " is added to a document to be replaced with a special document token |
| 90 | + documents = [". " + doc for doc in documents] |
| 91 | + encoded = self.tokenizer.encode_batch(documents) |
| 92 | + return encoded |
| 93 | + |
| 94 | + @classmethod |
| 95 | + def list_supported_models(cls) -> List[Dict[str, Any]]: |
| 96 | + """Lists the supported models. |
| 97 | +
|
| 98 | + Returns: |
| 99 | + List[Dict[str, Any]]: A list of dictionaries containing the model information. |
| 100 | + """ |
| 101 | + return supported_colbert_models |
| 102 | + |
| 103 | + def __init__( |
| 104 | + self, |
| 105 | + model_name: str, |
| 106 | + cache_dir: Optional[str] = None, |
| 107 | + threads: Optional[int] = None, |
| 108 | + providers: Optional[Sequence[OnnxProvider]] = None, |
| 109 | + **kwargs, |
| 110 | + ): |
| 111 | + """ |
| 112 | + Args: |
| 113 | + model_name (str): The name of the model to use. |
| 114 | + cache_dir (str, optional): The path to the cache directory. |
| 115 | + Can be set using the `FASTEMBED_CACHE_PATH` env variable. |
| 116 | + Defaults to `fastembed_cache` in the system's temp directory. |
| 117 | + threads (int, optional): The number of threads single onnxruntime session can use. Defaults to None. |
| 118 | +
|
| 119 | + Raises: |
| 120 | + ValueError: If the model_name is not in the format <org>/<model> e.g. BAAI/bge-base-en. |
| 121 | + """ |
| 122 | + |
| 123 | + super().__init__(model_name, cache_dir, threads, **kwargs) |
| 124 | + |
| 125 | + model_description = self._get_model_description(model_name) |
| 126 | + cache_dir = define_cache_dir(cache_dir) |
| 127 | + |
| 128 | + model_dir = self.download_model( |
| 129 | + model_description, cache_dir, local_files_only=self._local_files_only |
| 130 | + ) |
| 131 | + |
| 132 | + self.load_onnx_model( |
| 133 | + model_dir=model_dir, |
| 134 | + model_file=model_description["model_file"], |
| 135 | + threads=threads, |
| 136 | + providers=providers, |
| 137 | + ) |
| 138 | + self.mask_token_id = self.special_token_to_id["[MASK]"] |
| 139 | + self.pad_token_id = self.tokenizer.padding["pad_id"] |
| 140 | + |
| 141 | + self.skip_list = { |
| 142 | + self.tokenizer.encode(symbol, add_special_tokens=False).ids[0] |
| 143 | + for symbol in string.punctuation |
| 144 | + } |
| 145 | + |
| 146 | + def embed( |
| 147 | + self, |
| 148 | + documents: Union[str, Iterable[str]], |
| 149 | + batch_size: int = 256, |
| 150 | + parallel: Optional[int] = None, |
| 151 | + **kwargs, |
| 152 | + ) -> Iterable[np.ndarray]: |
| 153 | + """ |
| 154 | + Encode a list of documents into list of embeddings. |
| 155 | + We use mean pooling with attention so that the model can handle variable-length inputs. |
| 156 | +
|
| 157 | + Args: |
| 158 | + documents: Iterator of documents or single document to embed |
| 159 | + batch_size: Batch size for encoding -- higher values will use more memory, but be faster |
| 160 | + parallel: |
| 161 | + If > 1, data-parallel encoding will be used, recommended for offline encoding of large datasets. |
| 162 | + If 0, use all available cores. |
| 163 | + If None, don't use data-parallel processing, use default onnxruntime threading instead. |
| 164 | +
|
| 165 | + Returns: |
| 166 | + List of embeddings, one per document |
| 167 | + """ |
| 168 | + yield from self._embed_documents( |
| 169 | + model_name=self.model_name, |
| 170 | + cache_dir=str(self.cache_dir), |
| 171 | + documents=documents, |
| 172 | + batch_size=batch_size, |
| 173 | + parallel=parallel, |
| 174 | + ) |
| 175 | + |
| 176 | + def query_embed(self, query: Union[str, List[str]], **kwargs) -> np.ndarray: |
| 177 | + if isinstance(query, str): |
| 178 | + query = [query] |
| 179 | + |
| 180 | + for text in query: |
| 181 | + yield from self._post_process_onnx_output( |
| 182 | + self.onnx_embed([text], is_doc=False), is_doc=False |
| 183 | + ) |
| 184 | + |
| 185 | + @classmethod |
| 186 | + def _get_worker_class(cls) -> Type[TextEmbeddingWorker]: |
| 187 | + return ColbertEmbeddingWorker |
| 188 | + |
| 189 | + |
| 190 | +class ColbertEmbeddingWorker(TextEmbeddingWorker): |
| 191 | + def init_embedding( |
| 192 | + self, |
| 193 | + model_name: str, |
| 194 | + cache_dir: str, |
| 195 | + ) -> Colbert: |
| 196 | + return Colbert(model_name=model_name, cache_dir=cache_dir, threads=1) |
0 commit comments