Skip to content

Commit 1e298a0

Browse files
Anush008Anush008
authored
feat: Added gte-large, nomic-text 1.5, cleanup (#130)
1 parent 38c4eb1 commit 1e298a0

File tree

5 files changed

+82
-257
lines changed

5 files changed

+82
-257
lines changed

‎docs/examples/Supported_Models.ipynb

+59-35
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
},
1313
{
1414
"cell_type": "code",
15-
"execution_count": 2,
15+
"execution_count": 6,
1616
"metadata": {},
1717
"outputs": [
1818
{
@@ -110,46 +110,62 @@
110110
" </tr>\n",
111111
" <tr>\n",
112112
" <th>8</th>\n",
113-
" <td>sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2</td>\n",
114-
" <td>384</td>\n",
115-
" <td>Sentence Transformer model, paraphrase-multilingual-MiniLM-L12-v2</td>\n",
116-
" <td>0.46</td>\n",
117-
" <td>{'hf': 'qdrant/paraphrase-multilingual-MiniLM-L12-v2-onnx-Q'}</td>\n",
113+
" <td>nomic-ai/nomic-embed-text-v1</td>\n",
114+
" <td>768</td>\n",
115+
" <td>8192 context length english model</td>\n",
116+
" <td>0.54</td>\n",
117+
" <td>{'hf': 'nomic-ai/nomic-embed-text-v1'}</td>\n",
118118
" </tr>\n",
119119
" <tr>\n",
120120
" <th>9</th>\n",
121-
" <td>nomic-ai/nomic-embed-text-v1</td>\n",
121+
" <td>nomic-ai/nomic-embed-text-v1.5</td>\n",
122122
" <td>768</td>\n",
123123
" <td>8192 context length english model</td>\n",
124124
" <td>0.54</td>\n",
125-
" <td>{'hf': 'nomic-ai/nomic-embed-text-v1'}</td>\n",
125+
" <td>{'hf': 'nomic-ai/nomic-embed-text-v1.5'}</td>\n",
126126
" </tr>\n",
127127
" <tr>\n",
128128
" <th>10</th>\n",
129+
" <td>thenlper/gte-large</td>\n",
130+
" <td>1024</td>\n",
131+
" <td>Large general text embeddings model</td>\n",
132+
" <td>1.34</td>\n",
133+
" <td>{'hf': 'qdrant/gte-large-onnx'}</td>\n",
134+
" </tr>\n",
135+
" <tr>\n",
136+
" <th>11</th>\n",
129137
" <td>intfloat/multilingual-e5-large</td>\n",
130138
" <td>1024</td>\n",
131139
" <td>Multilingual model, e5-large. Recommend using this model for non-English languages</td>\n",
132140
" <td>2.24</td>\n",
133141
" <td>{'url': 'https://storage.googleapis.com/qdrant-fastembed/fast-multilingual-e5-large.tar.gz', 'hf': 'qdrant/multilingual-e5-large-onnx'}</td>\n",
134142
" </tr>\n",
135143
" <tr>\n",
136-
" <th>11</th>\n",
144+
" <th>12</th>\n",
137145
" <td>sentence-transformers/paraphrase-multilingual-mpnet-base-v2</td>\n",
138146
" <td>768</td>\n",
139147
" <td>Sentence-transformers model for tasks like clustering or semantic search</td>\n",
140148
" <td>1.11</td>\n",
141149
" <td>{'hf': 'xenova/paraphrase-multilingual-mpnet-base-v2'}</td>\n",
142150
" </tr>\n",
143151
" <tr>\n",
144-
" <th>12</th>\n",
152+
" <th>13</th>\n",
153+
" <td>sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2</td>\n",
154+
" <td>384</td>\n",
155+
" <td>Sentence Transformer model, paraphrase-multilingual-MiniLM-L12-v2</td>\n",
156+
" <td>0.46</td>\n",
157+
" <td>{'hf': 'qdrant/paraphrase-multilingual-MiniLM-L12-v2-onnx-Q'}</td>\n",
158+
" </tr>\n",
159+
" <tr>\n",
160+
" <th>14</th>\n",
145161
" <td>jinaai/jina-embeddings-v2-base-en</td>\n",
146162
" <td>768</td>\n",
147163
" <td>English embedding model supporting 8192 sequence length</td>\n",
148164
" <td>0.55</td>\n",
149165
" <td>{'hf': 'xenova/jina-embeddings-v2-base-en'}</td>\n",
150166
" </tr>\n",
151167
" <tr>\n",
152-
" <th>13</th>\n",
168+
" <th>15</th>\n",
153169
" <td>jinaai/jina-embeddings-v2-small-en</td>\n",
154170
" <td>512</td>\n",
155171
" <td>English embedding model supporting 8192 sequence length</td>\n",
@@ -170,12 +186,14 @@
170186
"5 BAAI/bge-small-en-v1.5 384 \n",
171187
"6 BAAI/bge-small-zh-v1.5 512 \n",
172188
"7 sentence-transformers/all-MiniLM-L6-v2 384 \n",
173-
"8 sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2 384 \n",
174-
"9 nomic-ai/nomic-embed-text-v1 768 \n",
175-
"10 intfloat/multilingual-e5-large 1024 \n",
176-
"11 sentence-transformers/paraphrase-multilingual-mpnet-base-v2 768 \n",
177-
"12 jinaai/jina-embeddings-v2-base-en 768 \n",
178-
"13 jinaai/jina-embeddings-v2-small-en 512 \n",
189+
"8 nomic-ai/nomic-embed-text-v1 768 \n",
190+
"9 nomic-ai/nomic-embed-text-v1.5 768 \n",
191+
"10 thenlper/gte-large 1024 \n",
192+
"11 intfloat/multilingual-e5-large 1024 \n",
193+
"12 sentence-transformers/paraphrase-multilingual-mpnet-base-v2 768 \n",
194+
"13 sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2 384 \n",
195+
"14 jinaai/jina-embeddings-v2-base-en 768 \n",
196+
"15 jinaai/jina-embeddings-v2-small-en 512 \n",
179197
"\n",
180198
" description \\\n",
181199
"0 Base English model \n",
@@ -186,12 +204,14 @@
186204
"5 Fast and Default English model \n",
187205
"6 Fast and recommended Chinese model \n",
188206
"7 Sentence Transformer model, MiniLM-L6-v2 \n",
189-
"8 Sentence Transformer model, paraphrase-multilingual-MiniLM-L12-v2 \n",
207+
"8 8192 context length english model \n",
190208
"9 8192 context length english model \n",
191-
"10 Multilingual model, e5-large. Recommend using this model for non-English languages \n",
192-
"11 Sentence-transformers model for tasks like clustering or semantic search \n",
193-
"12 English embedding model supporting 8192 sequence length \n",
194-
"13 English embedding model supporting 8192 sequence length \n",
209+
"10 Large general text embeddings model \n",
210+
"11 Multilingual model, e5-large. Recommend using this model for non-English languages \n",
211+
"12 Sentence-transformers model for tasks like clustering or semantic search \n",
212+
"13 Sentence Transformer model, paraphrase-multilingual-MiniLM-L12-v2 \n",
213+
"14 English embedding model supporting 8192 sequence length \n",
214+
"15 English embedding model supporting 8192 sequence length \n",
195215
"\n",
196216
" size_in_GB \\\n",
197217
"0 0.50 \n",
@@ -202,12 +222,14 @@
202222
"5 0.13 \n",
203223
"6 0.10 \n",
204224
"7 0.09 \n",
205-
"8 0.46 \n",
225+
"8 0.54 \n",
206226
"9 0.54 \n",
207-
"10 2.24 \n",
208-
"11 1.11 \n",
209-
"12 0.55 \n",
210-
"13 0.13 \n",
227+
"10 1.34 \n",
228+
"11 2.24 \n",
229+
"12 1.11 \n",
230+
"13 0.46 \n",
231+
"14 0.55 \n",
232+
"15 0.13 \n",
211233
"\n",
212234
" sources \n",
213235
"0 {'url': 'https://storage.googleapis.com/qdrant-fastembed/fast-bge-base-en.tar.gz'} \n",
@@ -218,15 +240,17 @@
218240
"5 {'url': 'https://storage.googleapis.com/qdrant-fastembed/fast-bge-small-en-v1.5.tar.gz', 'hf': 'qdrant/bge-small-en-v1.5-onnx-q'} \n",
219241
"6 {'url': 'https://storage.googleapis.com/qdrant-fastembed/fast-bge-small-zh-v1.5.tar.gz'} \n",
220242
"7 {'url': 'https://storage.googleapis.com/qdrant-fastembed/sentence-transformers-all-MiniLM-L6-v2.tar.gz', 'hf': 'qdrant/all-MiniLM-L6-v2-onnx'} \n",
221-
"8 {'hf': 'qdrant/paraphrase-multilingual-MiniLM-L12-v2-onnx-Q'} \n",
222-
"9 {'hf': 'nomic-ai/nomic-embed-text-v1'} \n",
223-
"10 {'url': 'https://storage.googleapis.com/qdrant-fastembed/fast-multilingual-e5-large.tar.gz', 'hf': 'qdrant/multilingual-e5-large-onnx'} \n",
224-
"11 {'hf': 'xenova/paraphrase-multilingual-mpnet-base-v2'} \n",
225-
"12 {'hf': 'xenova/jina-embeddings-v2-base-en'} \n",
226-
"13 {'hf': 'xenova/jina-embeddings-v2-small-en'} "
243+
"8 {'hf': 'nomic-ai/nomic-embed-text-v1'} \n",
244+
"9 {'hf': 'nomic-ai/nomic-embed-text-v1.5'} \n",
245+
"10 {'hf': 'qdrant/gte-large-onnx'} \n",
246+
"11 {'url': 'https://storage.googleapis.com/qdrant-fastembed/fast-multilingual-e5-large.tar.gz', 'hf': 'qdrant/multilingual-e5-large-onnx'} \n",
247+
"12 {'hf': 'xenova/paraphrase-multilingual-mpnet-base-v2'} \n",
248+
"13 {'hf': 'qdrant/paraphrase-multilingual-MiniLM-L12-v2-onnx-Q'} \n",
249+
"14 {'hf': 'xenova/jina-embeddings-v2-base-en'} \n",
250+
"15 {'hf': 'xenova/jina-embeddings-v2-small-en'} "
227251
]
228252
},
229-
"execution_count": 2,
253+
"execution_count": 6,
230254
"metadata": {},
231255
"output_type": "execute_result"
232256
}
@@ -256,7 +280,7 @@
256280
"name": "python",
257281
"nbconvert_exporter": "python",
258282
"pygments_lexer": "ipython3",
259-
"version": "3.11.4"
283+
"version": "3.10.13"
260284
},
261285
"orig_nbformat": 4
262286
},

‎fastembed/models.json

-143
This file was deleted.

‎fastembed/text/onnx_embedding.py

+18
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,24 @@
116116
"hf": "nomic-ai/nomic-embed-text-v1",
117117
},
118118
},
119+
{
120+
"model": "nomic-ai/nomic-embed-text-v1.5",
121+
"dim": 768,
122+
"description": "8192 context length english model",
123+
"size_in_GB": 0.54,
124+
"sources": {
125+
"hf": "nomic-ai/nomic-embed-text-v1.5",
126+
},
127+
},
128+
{
129+
"model": "thenlper/gte-large",
130+
"dim": 1024,
131+
"description": "Large general text embeddings model",
132+
"size_in_GB": 1.34,
133+
"sources": {
134+
"hf": "qdrant/gte-large-onnx",
135+
},
136+
},
119137
# {
120138
# "model": "sentence-transformers/all-MiniLM-L6-v2",
121139
# "dim": 384,

0 commit comments

Comments
 (0)