support lmms-eval

Author: chengtao-lv

configs/quantization/methods/RTN/rtn_w_a_lmms_eval.yml

@@ -0,0 +1,29 @@
+base:
+    seed: &seed 42
+model:
+    type: model_type
+    path: model path
+    torch_dtype: auto
+eval:
+    eval_pos: [pretrain, fake_quant]
+    type: vqa
+    name: mme
+    download: False
+    path: MME dataset path
+    bs: 1
+    inference_per_block: False
+quant:
+    method: RTN
+    quant_objects: [vision, language] # default is [language]
+    weight:
+        bit: 8
+        symmetric: True
+        granularity: per_channel
+        group_size: -1
+    act:
+        bit: 8
+        symmetric: True
+        granularity: per_token
+save:
+    save_fake: False
+    save_path: /path/to/save/

llmc/__main__.py

@@ -48,9 +48,6 @@ def main(config):
                 if config.eval.type == 'acc':
                     acc_eval = AccuracyEval(config_for_eval)
                     eval_list.append(acc_eval)
-                elif config.eval.type == 'img_txt':
-                    acc_eval = VLMEval(config_for_eval)
-                    eval_list.append(acc_eval)
                 elif config.eval.type == 'vqa':
                     vqa_eval = VQAEval(config_for_eval)
                     eval_list.append(vqa_eval)
@@ -66,10 +63,6 @@ def main(config):
                 for acc_eval in eval_list:
                     acc = acc_eval.eval(model)
                     logger.info(f'{config.eval.name} acc : {acc}')
-            elif config.eval.type == 'img_txt':
-                for vlm_eval in eval_list:
-                    results = vlm_eval.eval(model)
-                    logger.info(f'{config.eval.name} results : {results}')
             elif config.eval.type == 'vqa':
                 for vqa_eval in eval_list:
                     results = vqa_eval.eval(model)
@@ -135,10 +128,6 @@ def main(config):
                 for acc_eval in eval_list:
                     acc = acc_eval.eval(model)
                     logger.info(f'{config.eval.name} acc : {acc}')
-            elif config.eval.type == 'img_txt':
-                for vlm_eval in eval_list:
-                    results = vlm_eval.eval(model)
-                    logger.info(f'{config.eval.name} results : {results}')
             elif config.eval.type == 'code' and config.eval.name == 'human_eval':
                 for human_eval in eval_list:
                     results = human_eval.eval(model, eval_pos='transformed')
@@ -167,10 +156,7 @@ def main(config):
                 for acc_eval in eval_list:
                     acc = acc_eval.eval(model)
                     logger.info(f'{config.eval.name} acc : {acc}')
-            elif config.eval.type == 'img_txt':
-                for vlm_eval in eval_list:
-                    results = vlm_eval.eval(model)
-                    logger.info(f'{config.eval.name} results : {results}')
+
             elif config.eval.type == 'vqa':
                 for vqa_eval in eval_list:
                     results = vqa_eval.eval(model)

llmc/eval/__init__.py

@@ -2,5 +2,4 @@
 from .eval_code import HumanEval
 from .eval_ppl import PerplexityEval
 from .eval_token_consist import TokenConsistencyEval
-from .eval_vlm import VLMEval
 from .eval_vqa import VQAEval