Merge pull request #34 from codefuse-ai/modelcache_dev_mm

Correct the format

Author: peng3307165

modelcache/embedding/data2vec.py

@@ -8,7 +8,7 @@
 
 
 def mean_pooling(model_output, attention_mask):
-    token_embeddings = model_output[0]  # First element of model_output contains all token embeddings
+    token_embeddings = model_output[0]
     input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
     return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
 

modelcache_mm/adapter/adapter_query.py

@@ -47,37 +47,36 @@ def adapt_query(cache_data_convert, *args, **kwargs):
     else:
         raise MultiTypeError
 
-    embedding_data = None
-    mm_type = None
-    if cache_enable:
-        if pre_multi_type == 'IMG_TEXT':
-            embedding_data_resp = time_cal(
-                chat_cache.embedding_func,
-                func_name="iat_embedding",
-                report_func=chat_cache.report.embedding,
-            )(data_dict)
-        else:
-            embedding_data_resp = time_cal(
-                chat_cache.embedding_func,
-                func_name="iat_embedding",
-                report_func=chat_cache.report.embedding,
-            )(data_dict)
-        image_embeddings = embedding_data_resp['image_embedding']
-        text_embeddings = embedding_data_resp['text_embeddings']
-
-        if len(image_embeddings) > 0 and len(image_embeddings) > 0:
-            embedding_data = np.concatenate((image_embeddings, text_embeddings))
-            mm_type = 'mm'
-        elif len(image_embeddings) > 0:
-            image_embedding = np.array(image_embeddings[0])
-            embedding_data = image_embedding
-            mm_type = 'image'
-        elif len(text_embeddings) > 0:
-            text_embedding = np.array(text_embeddings[0])
-            embedding_data = text_embedding
-            mm_type = 'text'
-        else:
-            raise ValueError('maya embedding service return both empty list, please check!')
+    # embedding_data = None
+    # mm_type = None
+    if pre_multi_type == 'IMG_TEXT':
+        embedding_data_resp = time_cal(
+            chat_cache.embedding_func,
+            func_name="mm_embedding",
+            report_func=chat_cache.report.embedding,
+        )(data_dict)
+    else:
+        embedding_data_resp = time_cal(
+            chat_cache.embedding_func,
+            func_name="mm_embedding",
+            report_func=chat_cache.report.embedding,
+        )(data_dict)
+    image_embeddings = embedding_data_resp['image_embedding']
+    text_embeddings = embedding_data_resp['text_embeddings']
+
+    if len(image_embeddings) > 0 and len(image_embeddings) > 0:
+        embedding_data = np.concatenate((image_embeddings, text_embeddings))
+        # mm_type = 'mm'
+    elif len(image_embeddings) > 0:
+        image_embedding = np.array(image_embeddings[0])
+        embedding_data = image_embedding
+        # mm_type = 'image'
+    elif len(text_embeddings) > 0:
+        text_embedding = np.array(text_embeddings[0])
+        embedding_data = text_embedding
+        # mm_type = 'text'
+    else:
+        raise ValueError('maya embedding service return both empty list, please check!')
 
     if cache_enable:
         cache_data_list = time_cal(

modelcache_mm/embedding/clip.py

@@ -18,8 +18,8 @@ def to_embeddings(self, data_dict, **_):
         text_list = data_dict['text']
         image_data = data_dict['image']
 
-        img_data = None
-        txt_data = None
+        # img_data = None
+        # txt_data = None
 
         if image_data:
             input_img = load_image(image_data)
@@ -46,8 +46,4 @@ def post_proc(self, token_embeddings, inputs):
 
     @property
     def dimension(self):
-        """Embedding dimension.
-
-        :return: embedding dimension
-        """
         return self.__dimension

modelcache_mm/manager/data_manager.py

@@ -34,7 +34,7 @@ def save_query_resp(self, query_resp_dict, **kwargs):
     @abstractmethod
     def import_data(
         self, texts: List[Any], image_urls: List[Any], image_ids: List[Any], answers: List[Answer],
-            embeddings: List[Any], model: Any, iat_type: Any
+            embeddings: List[Any], model: Any, mm_type: Any
     ):
         pass
 
@@ -96,7 +96,7 @@ def save_query_resp(self, query_resp_dict, **kwargs):
 
     def import_data(
         self, texts: List[Any], image_urls: List[Any], image_ids: List[Any], answers: List[Answer],
-            embeddings: List[Any], model: Any, iat_type: Any
+            embeddings: List[Any], model: Any, mm_type: Any
     ):
         pass