chore: Added license for all models (#364)

hh-space-invader · web-flow · commit 0638c011dc3b · 2024-10-17T08:57:33.000+02:00
* chore: Added license for all models

* chore: added license as key value pair

* chore: Updated clip models text/image license
diff --git a/fastembed/image/image_embedding.py b/fastembed/image/image_embedding.py
@@ -25,6 +25,7 @@ def list_supported_models(cls) -> List[Dict[str, Any]]:
                         "model": "Qdrant/clip-ViT-B-32-vision",
                         "dim": 512,
                         "description": "CLIP vision encoder based on ViT-B/32",
+                        "license": "mit",
                         "size_in_GB": 0.33,
                         "sources": {
                             "hf": "Qdrant/clip-ViT-B-32-vision",
diff --git a/fastembed/image/onnx_embedding.py b/fastembed/image/onnx_embedding.py
@@ -13,6 +13,7 @@
         "model": "Qdrant/clip-ViT-B-32-vision",
         "dim": 512,
         "description": "Image embeddings, Multimodal (text&image), 2021 year",
+        "license": "mit",
         "size_in_GB": 0.34,
         "sources": {
             "hf": "Qdrant/clip-ViT-B-32-vision",
@@ -23,6 +24,7 @@
         "model": "Qdrant/resnet50-onnx",
         "dim": 2048,
         "description": "Image embeddings, Unimodal (image), 2016 year",
+        "license": "apache-2.0",
         "size_in_GB": 0.1,
         "sources": {
             "hf": "Qdrant/resnet50-onnx",
@@ -33,6 +35,7 @@
         "model": "Qdrant/Unicom-ViT-B-16",
         "dim": 768,
         "description": "Image embeddings (more detailed than Unicom-ViT-B-32), Multimodal (text&image), 2023 year",
+        "license": "apache-2.0",
         "size_in_GB": 0.82,
         "sources": {
             "hf": "Qdrant/Unicom-ViT-B-16",
@@ -43,6 +46,7 @@
         "model": "Qdrant/Unicom-ViT-B-32",
         "dim": 512,
         "description": "Image embeddings, Multimodal (text&image), 2023 year",
+        "license": "apache-2.0",
         "size_in_GB": 0.48,
         "sources": {
             "hf": "Qdrant/Unicom-ViT-B-32",
diff --git a/fastembed/late_interaction/colbert.py b/fastembed/late_interaction/colbert.py
@@ -17,6 +17,7 @@
         "model": "colbert-ir/colbertv2.0",
         "dim": 128,
         "description": "Late interaction model",
+        "license": "mit",
         "size_in_GB": 0.44,
         "sources": {
             "hf": "colbert-ir/colbertv2.0",
@@ -27,6 +28,7 @@
         "model": "answerdotai/answerai-colbert-small-v1",
         "dim": 96,
         "description": "Text embeddings, Unimodal (text), Multilingual (~100 languages), 512 input tokens truncation, 2024 year",
+        "license": "apache-2.0",
         "size_in_GB": 0.13,
         "sources": {
             "hf": "answerdotai/answerai-colbert-small-v1",
diff --git a/fastembed/late_interaction/late_interaction_text_embedding.py b/fastembed/late_interaction/late_interaction_text_embedding.py
@@ -26,14 +26,16 @@ def list_supported_models(cls) -> List[Dict[str, Any]]:
                 ```
                 [
                     {
-                        "model": "prithvida/SPLADE_PP_en_v1",
-                        "vocab_size": 30522,
-                        "description": "Independent Implementation of SPLADE++ Model for English",
-                        "size_in_GB": 0.532,
+                        "model": "colbert-ir/colbertv2.0",
+                        "dim": 128,
+                        "description": "Late interaction model",
+                        "license": "mit",
+                        "size_in_GB": 0.44,
                         "sources": {
-                            "hf": "qdrant/SPLADE_PP_en_v1",
+                            "hf": "colbert-ir/colbertv2.0",
                         },
-                    }
+                        "model_file": "model.onnx",
+                    },
                 ]
                 ```
         """
diff --git a/fastembed/rerank/cross_encoder/onnx_text_cross_encoder.py b/fastembed/rerank/cross_encoder/onnx_text_cross_encoder.py
@@ -16,6 +16,7 @@
         },
         "model_file": "onnx/model.onnx",
         "description": "MiniLM-L-6-v2 model optimized for re-ranking tasks.",
+        "license": "apache-2.0",
     },
     {
         "model": "Xenova/ms-marco-MiniLM-L-12-v2",
@@ -25,6 +26,7 @@
         },
         "model_file": "onnx/model.onnx",
         "description": "MiniLM-L-12-v2 model optimized for re-ranking tasks.",
+        "license": "apache-2.0",
     },
     {
         "model": "BAAI/bge-reranker-base",
@@ -34,6 +36,7 @@
         },
         "model_file": "onnx/model.onnx",
         "description": "BGE reranker base model for cross-encoder re-ranking.",
+        "license": "mit",
     },
 ]
 
diff --git a/fastembed/rerank/cross_encoder/text_cross_encoder.py b/fastembed/rerank/cross_encoder/text_cross_encoder.py
@@ -28,6 +28,7 @@ def list_supported_models(cls) -> List[Dict[str, Any]]:
                         },
                         "model_file": "onnx/model.onnx",
                         "description": "MiniLM-L-6-v2 model optimized for re-ranking tasks.",
+                        "license": "apache-2.0",
                     }
                 ]
                 ```
diff --git a/fastembed/sparse/bm25.py b/fastembed/sparse/bm25.py
@@ -52,6 +52,7 @@
     {
         "model": "Qdrant/bm25",
         "description": "BM25 as sparse embeddings meant to be used with Qdrant",
+        "license": "apache-2.0",
         "size_in_GB": 0.01,
         "sources": {
             "hf": "Qdrant/bm25",
diff --git a/fastembed/sparse/bm42.py b/fastembed/sparse/bm42.py
@@ -21,6 +21,7 @@
         "model": "Qdrant/bm42-all-minilm-l6-v2-attentions",
         "vocab_size": 30522,
         "description": "Light sparse embedding model, which assigns an importance score to each token in the text",
+        "license": "apache-2.0",
         "size_in_GB": 0.09,
         "sources": {
             "hf": "Qdrant/all_miniLM_L6_v2_with_attentions",
diff --git a/fastembed/sparse/sparse_text_embedding.py b/fastembed/sparse/sparse_text_embedding.py
@@ -29,6 +29,7 @@ def list_supported_models(cls) -> List[Dict[str, Any]]:
                         "model": "prithvida/SPLADE_PP_en_v1",
                         "vocab_size": 30522,
                         "description": "Independent Implementation of SPLADE++ Model for English",
+                        "license": "apache-2.0",
                         "size_in_GB": 0.532,
                         "sources": {
                             "hf": "qdrant/SPLADE_PP_en_v1",
diff --git a/fastembed/sparse/splade_pp.py b/fastembed/sparse/splade_pp.py
@@ -14,7 +14,8 @@
     {
         "model": "prithivida/Splade_PP_en_v1",
         "vocab_size": 30522,
-        "description": "Independent Implementation of SPLADE++ Model for English",
+        "description": "Independent Implementation of SPLADE++ Model for English.",
+        "license": "apache-2.0",
         "size_in_GB": 0.532,
         "sources": {
             "hf": "Qdrant/SPLADE_PP_en_v1",
@@ -24,7 +25,8 @@
     {
         "model": "prithvida/Splade_PP_en_v1",
         "vocab_size": 30522,
-        "description": "Independent Implementation of SPLADE++ Model for English",
+        "description": "Independent Implementation of SPLADE++ Model for English.",
+        "license": "apache-2.0",
         "size_in_GB": 0.532,
         "sources": {
             "hf": "Qdrant/SPLADE_PP_en_v1",
diff --git a/fastembed/text/clip_embedding.py b/fastembed/text/clip_embedding.py
@@ -11,6 +11,7 @@
         "model": "Qdrant/clip-ViT-B-32-text",
         "dim": 512,
         "description": "Text embeddings, Multimodal (text&image), English, 77 input tokens truncation, Prefixes for queries/documents: not necessary, 2021 year",
+        "license": "mit",
         "size_in_GB": 0.25,
         "sources": {
             "hf": "Qdrant/clip-ViT-B-32-text",
diff --git a/fastembed/text/e5_onnx_embedding.py b/fastembed/text/e5_onnx_embedding.py
@@ -9,7 +9,8 @@
     {
         "model": "intfloat/multilingual-e5-large",
         "dim": 1024,
-        "description": "Text embeddings, Unimodal (text), Multilingual (~100 languages), 512 input tokens truncation, Prefixes for queries/documents: necessary, 2024 year",
+        "description": "Text embeddings, Unimodal (text), Multilingual (~100 languages), 512 input tokens truncation, Prefixes for queries/documents: necessary, 2024 year.",
+        "license": "mit",
         "size_in_GB": 2.24,
         "sources": {
             "url": "https://storage.googleapis.com/qdrant-fastembed/fast-multilingual-e5-large.tar.gz",
@@ -21,7 +22,8 @@
     {
         "model": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2",
         "dim": 768,
-        "description": "Text embeddings, Unimodal (text), Multilingual (~50 languages), 384 input tokens truncation, Prefixes for queries/documents: not necessary, 2021 year",
+        "description": "Text embeddings, Unimodal (text), Multilingual (~50 languages), 384 input tokens truncation, Prefixes for queries/documents: not necessary, 2021 year.",
+        "license": "apache-2.0",
         "size_in_GB": 1.00,
         "sources": {
             "hf": "xenova/paraphrase-multilingual-mpnet-base-v2",
diff --git a/fastembed/text/onnx_embedding.py b/fastembed/text/onnx_embedding.py
@@ -12,7 +12,8 @@
     {
         "model": "BAAI/bge-base-en",
         "dim": 768,
-        "description": "Text embeddings, Unimodal (text), English, 512 input tokens truncation, Prefixes for queries/documents: necessary, 2023 year",
+        "description": "Text embeddings, Unimodal (text), English, 512 input tokens truncation, Prefixes for queries/documents: necessary, 2023 year.",
+        "license": "mit",
         "size_in_GB": 0.42,
         "sources": {
             "url": "https://storage.googleapis.com/qdrant-fastembed/fast-bge-base-en.tar.gz",
@@ -22,7 +23,8 @@
     {
         "model": "BAAI/bge-base-en-v1.5",
         "dim": 768,
-        "description": "Text embeddings, Unimodal (text), English, 512 input tokens truncation, Prefixes for queries/documents: not so necessary, 2023 year",
+        "description": "Text embeddings, Unimodal (text), English, 512 input tokens truncation, Prefixes for queries/documents: not so necessary, 2023 year.",
+        "license": "mit",
         "size_in_GB": 0.21,
         "sources": {
             "url": "https://storage.googleapis.com/qdrant-fastembed/fast-bge-base-en-v1.5.tar.gz",
@@ -33,7 +35,8 @@
     {
         "model": "BAAI/bge-large-en-v1.5",
         "dim": 1024,
-        "description": "Text embeddings, Unimodal (text), English, 512 input tokens truncation, Prefixes for queries/documents: not so necessary, 2023 year",
+        "description": "Text embeddings, Unimodal (text), English, 512 input tokens truncation, Prefixes for queries/documents: not so necessary, 2023 year.",
+        "license": "mit",
         "size_in_GB": 1.20,
         "sources": {
             "hf": "qdrant/bge-large-en-v1.5-onnx",
@@ -43,7 +46,8 @@
     {
         "model": "BAAI/bge-small-en",
         "dim": 384,
-        "description": "Text embeddings, Unimodal (text), English, 512 input tokens truncation, Prefixes for queries/documents: necessary, 2023 year",
+        "description": "Text embeddings, Unimodal (text), English, 512 input tokens truncation, Prefixes for queries/documents: necessary, 2023 year.",
+        "license": "mit",
         "size_in_GB": 0.13,
         "sources": {
             "url": "https://storage.googleapis.com/qdrant-fastembed/BAAI-bge-small-en.tar.gz",
@@ -53,7 +57,8 @@
     {
         "model": "BAAI/bge-small-en-v1.5",
         "dim": 384,
-        "description": "Text embeddings, Unimodal (text), English, 512 input tokens truncation, Prefixes for queries/documents: not so necessary, 2023 year",
+        "description": "Text embeddings, Unimodal (text), English, 512 input tokens truncation, Prefixes for queries/documents: not so necessary, 2023 year.",
+        "license": "mit",
         "size_in_GB": 0.067,
         "sources": {
             "hf": "qdrant/bge-small-en-v1.5-onnx-q",
@@ -63,7 +68,8 @@
     {
         "model": "BAAI/bge-small-zh-v1.5",
         "dim": 512,
-        "description": "Text embeddings, Unimodal (text), Chinese, 512 input tokens truncation, Prefixes for queries/documents: not so necessary, 2023 year",
+        "description": "Text embeddings, Unimodal (text), Chinese, 512 input tokens truncation, Prefixes for queries/documents: not so necessary, 2023 year.",
+        "license": "mit",
         "size_in_GB": 0.09,
         "sources": {
             "url": "https://storage.googleapis.com/qdrant-fastembed/fast-bge-small-zh-v1.5.tar.gz",
@@ -73,7 +79,8 @@
     {
         "model": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2",
         "dim": 384,
-        "description": "Text embeddings, Unimodal (text), Multilingual (~50 languages), 512 input tokens truncation, Prefixes for queries/documents: not necessary, 2019 year",
+        "description": "Text embeddings, Unimodal (text), Multilingual (~50 languages), 512 input tokens truncation, Prefixes for queries/documents: not necessary, 2019 year.",
+        "license": "apache-2.0",
         "size_in_GB": 0.22,
         "sources": {
             "hf": "qdrant/paraphrase-multilingual-MiniLM-L12-v2-onnx-Q",
@@ -83,7 +90,8 @@
     {
         "model": "thenlper/gte-large",
         "dim": 1024,
-        "description": "Text embeddings, Unimodal (text), English, 512 input tokens truncation, Prefixes for queries/documents: not necessary, 2023 year",
+        "description": "Text embeddings, Unimodal (text), English, 512 input tokens truncation, Prefixes for queries/documents: not necessary, 2023 year.",
+        "license": "mit",
         "size_in_GB": 1.20,
         "sources": {
             "hf": "qdrant/gte-large-onnx",
@@ -93,7 +101,8 @@
     {
         "model": "mixedbread-ai/mxbai-embed-large-v1",
         "dim": 1024,
-        "description": "Text embeddings, Unimodal (text), English, 512 input tokens truncation, Prefixes for queries/documents: necessary, 2024 year",
+        "description": "Text embeddings, Unimodal (text), English, 512 input tokens truncation, Prefixes for queries/documents: necessary, 2024 year.",
+        "license": "apache-2.0",
         "size_in_GB": 0.64,
         "sources": {
             "hf": "mixedbread-ai/mxbai-embed-large-v1",
@@ -103,7 +112,8 @@
     {
         "model": "snowflake/snowflake-arctic-embed-xs",
         "dim": 384,
-        "description": "Text embeddings, Unimodal (text), English, 512 input tokens truncation, Prefixes for queries/documents: necessary, 2024 year",
+        "description": "Text embeddings, Unimodal (text), English, 512 input tokens truncation, Prefixes for queries/documents: necessary, 2024 year.",
+        "license": "apache-2.0",
         "size_in_GB": 0.09,
         "sources": {
             "hf": "snowflake/snowflake-arctic-embed-xs",
@@ -113,7 +123,8 @@
     {
         "model": "snowflake/snowflake-arctic-embed-s",
         "dim": 384,
-        "description": "Text embeddings, Unimodal (text), English, 512 input tokens truncation, Prefixes for queries/documents: necessary, 2024 year",
+        "description": "Text embeddings, Unimodal (text), English, 512 input tokens truncation, Prefixes for queries/documents: necessary, 2024 year.",
+        "license": "apache-2.0",
         "size_in_GB": 0.13,
         "sources": {
             "hf": "snowflake/snowflake-arctic-embed-s",
@@ -123,7 +134,8 @@
     {
         "model": "snowflake/snowflake-arctic-embed-m",
         "dim": 768,
-        "description": "Text embeddings, Unimodal (text), English, 512 input tokens truncation, Prefixes for queries/documents: necessary, 2024 year",
+        "description": "Text embeddings, Unimodal (text), English, 512 input tokens truncation, Prefixes for queries/documents: necessary, 2024 year.",
+        "license": "apache-2.0",
         "size_in_GB": 0.43,
         "sources": {
             "hf": "Snowflake/snowflake-arctic-embed-m",
@@ -133,7 +145,8 @@
     {
         "model": "snowflake/snowflake-arctic-embed-m-long",
         "dim": 768,
-        "description": "Text embeddings, Unimodal (text), English, 2048 input tokens truncation, Prefixes for queries/documents: necessary, 2024 year",
+        "description": "Text embeddings, Unimodal (text), English, 2048 input tokens truncation, Prefixes for queries/documents: necessary, 2024 year.",
+        "license": "apache-2.0",
         "size_in_GB": 0.54,
         "sources": {
             "hf": "snowflake/snowflake-arctic-embed-m-long",
@@ -143,7 +156,8 @@
     {
         "model": "snowflake/snowflake-arctic-embed-l",
         "dim": 1024,
-        "description": "Text embeddings, Unimodal (text), English, 512 input tokens truncation, Prefixes for queries/documents: necessary, 2024 year",
+        "description": "Text embeddings, Unimodal (text), English, 512 input tokens truncation, Prefixes for queries/documents: necessary, 2024 year.",
+        "license": "apache-2.0",
         "size_in_GB": 1.02,
         "sources": {
             "hf": "snowflake/snowflake-arctic-embed-l",
diff --git a/fastembed/text/pooled_embedding.py b/fastembed/text/pooled_embedding.py
@@ -10,7 +10,8 @@
     {
         "model": "nomic-ai/nomic-embed-text-v1.5",
         "dim": 768,
-        "description": "Text embeddings, Multimodal (text, image), English, 8192 input tokens truncation, Prefixes for queries/documents: necessary, 2024 year",
+        "description": "Text embeddings, Multimodal (text, image), English, 8192 input tokens truncation, Prefixes for queries/documents: necessary, 2024 year.",
+        "license": "apache-2.0",
         "size_in_GB": 0.52,
         "sources": {
             "hf": "nomic-ai/nomic-embed-text-v1.5",
@@ -20,7 +21,8 @@
     {
         "model": "nomic-ai/nomic-embed-text-v1.5-Q",
         "dim": 768,
-        "description": "Text embeddings, Multimodal (text, image), English, 8192 input tokens truncation, Prefixes for queries/documents: necessary, 2024 year",
+        "description": "Text embeddings, Multimodal (text, image), English, 8192 input tokens truncation, Prefixes for queries/documents: necessary, 2024 year.",
+        "license": "apache-2.0",
         "size_in_GB": 0.13,
         "sources": {
             "hf": "nomic-ai/nomic-embed-text-v1.5",
@@ -30,7 +32,8 @@
     {
         "model": "nomic-ai/nomic-embed-text-v1",
         "dim": 768,
-        "description": "Text embeddings, Multimodal (text, image), English, 8192 input tokens truncation, Prefixes for queries/documents: necessary, 2024 year",
+        "description": "Text embeddings, Multimodal (text, image), English, 8192 input tokens truncation, Prefixes for queries/documents: necessary, 2024 year.",
+        "license": "apache-2.0",
         "size_in_GB": 0.52,
         "sources": {
             "hf": "nomic-ai/nomic-embed-text-v1",
diff --git a/fastembed/text/pooled_normalized_embedding.py b/fastembed/text/pooled_normalized_embedding.py
diff --git a/fastembed/text/text_embedding.py b/fastembed/text/text_embedding.py

Original file line number	Diff line number	Diff line change
`@@ -28,6 +28,7 @@ def list_supported_models(cls) -> List[Dict[str, Any]]:`
`28`	`28`	`},`
`29`	`29`	`"model_file": "onnx/model.onnx",`
`30`	`30`	`"description": "MiniLM-L-6-v2 model optimized for re-ranking tasks.",`
	`31`	`+ "license": "apache-2.0",`
`31`	`32`	`}`
`32`	`33`	`]`
`33`	`34`	```
Original file line number	Diff line number	Diff line change
`@@ -52,6 +52,7 @@`
`52`	`52`	`{`
`53`	`53`	`"model": "Qdrant/bm25",`
`54`	`54`	`"description": "BM25 as sparse embeddings meant to be used with Qdrant",`
	`55`	`+ "license": "apache-2.0",`
`55`	`56`	`"size_in_GB": 0.01,`
`56`	`57`	`"sources": {`
`57`	`58`	`"hf": "Qdrant/bm25",`