support lmms-eval (#246)

Author: chengtao-lv

configs/quantization/methods/Awq/awq_w_only_vlm.yml

@@ -18,11 +18,11 @@ calib:
     seed: *seed
 eval:
     eval_pos: [pretrain, fake_quant]
-    type: img_txt
-    name: MME
+    type: vqa
+    name: mme
     download: False
     path: MME dataset path
-    bs: 16
+    bs: 1
     inference_per_block: False
 quant:
     method: Awq

configs/quantization/methods/GPTQ/gptq_w_only_vlm.yml

@@ -18,11 +18,11 @@ calib:
     seed: *seed
 eval:
     eval_pos: [pretrain, fake_quant]
-    type: img_txt
-    name: MME
+    type: vqa
+    name: mme
     download: False
     path: MME dataset path
-    bs: 16
+    bs: 1
     inference_per_block: False
 quant:
     method: GPTQ

configs/quantization/methods/RTN/rtn_w_a_vlm.yml

@@ -6,11 +6,11 @@ model:
     torch_dtype: auto
 eval:
     eval_pos: [pretrain, fake_quant]
-    type: img_txt
-    name: MME
+    type: vqa
+    name: mme
     download: False
     path: MME dataset path
-    bs: 16
+    bs: 1
     inference_per_block: False
 quant:
     method: RTN

configs/quantization/methods/SmoothQuant/smoothquant_w_a_vlm.yml

@@ -18,11 +18,11 @@ calib:
     seed: *seed
 eval:
     eval_pos: [pretrain, fake_quant]
-    type: img_txt
-    name: MME
+    type: vqa
+    name: mme
     download: False
     path: MME dataset path
-    bs: 16
+    bs: 1
     inference_per_block: False
 quant:
     method: SmoothQuant

llmc/__main__.py

@@ -3,20 +3,22 @@
 import gc
 import json
 import os
+import sys
 import time
 
 import torch
 import torch.distributed as dist
 import yaml
 from easydict import EasyDict
+from lmms_eval.utils import make_table
 from loguru import logger
 from torch.distributed import destroy_process_group, init_process_group
 
 from llmc.compression.quantization import *
 from llmc.compression.sparsification import *
 from llmc.data import BaseDataset
 from llmc.eval import (AccuracyEval, HumanEval, PerplexityEval,
-                       TokenConsistencyEval, VLMEval)
+                       TokenConsistencyEval, VQAEval)
 from llmc.models import *
 from llmc.utils import (check_config, mkdirs, print_important_package_version,
                         seed_all, update_autoawq_quant_config,
@@ -46,9 +48,9 @@ def main(config):
                 if config.eval.type == 'acc':
                     acc_eval = AccuracyEval(config_for_eval)
                     eval_list.append(acc_eval)
-                elif config.eval.type == 'img_txt':
-                    acc_eval = VLMEval(config_for_eval)
-                    eval_list.append(acc_eval)
+                elif config.eval.type == 'vqa':
+                    vqa_eval = VQAEval(config_for_eval)
+                    eval_list.append(vqa_eval)
                 elif config.eval.type == 'code' and config.eval.name == 'human_eval':
                     human_eval = HumanEval(model.get_tokenizer(), config_for_eval)
                     eval_list.append(human_eval)
@@ -61,10 +63,11 @@ def main(config):
                 for acc_eval in eval_list:
                     acc = acc_eval.eval(model)
                     logger.info(f'{config.eval.name} acc : {acc}')
-            elif config.eval.type == 'img_txt':
-                for vlm_eval in eval_list:
-                    results = vlm_eval.eval(model)
-                    logger.info(f'{config.eval.name} results : {results}')
+            elif config.eval.type == 'vqa':
+                for vqa_eval in eval_list:
+                    results = vqa_eval.eval(model)
+                    logger.info(f'{config.eval.name} results :')
+                    print(make_table(results))
             elif config.eval.type == 'code' and config.eval.name == 'human_eval':
                 for human_eval in eval_list:
                     results = human_eval.eval(model, eval_pos='pretrain')
@@ -125,10 +128,6 @@ def main(config):
                 for acc_eval in eval_list:
                     acc = acc_eval.eval(model)
                     logger.info(f'{config.eval.name} acc : {acc}')
-            elif config.eval.type == 'img_txt':
-                for vlm_eval in eval_list:
-                    results = vlm_eval.eval(model)
-                    logger.info(f'{config.eval.name} results : {results}')
             elif config.eval.type == 'code' and config.eval.name == 'human_eval':
                 for human_eval in eval_list:
                     results = human_eval.eval(model, eval_pos='transformed')
@@ -157,10 +156,12 @@ def main(config):
                 for acc_eval in eval_list:
                     acc = acc_eval.eval(model)
                     logger.info(f'{config.eval.name} acc : {acc}')
-            elif config.eval.type == 'img_txt':
-                for vlm_eval in eval_list:
-                    results = vlm_eval.eval(model)
-                    logger.info(f'{config.eval.name} results : {results}')
+
+            elif config.eval.type == 'vqa':
+                for vqa_eval in eval_list:
+                    results = vqa_eval.eval(model)
+                    logger.info(f'{config.eval.name} results :')
+                    print(make_table(results))
             elif config.eval.type == 'code' and config.eval.name == 'human_eval':
                 for human_eval in eval_list:
                     results = human_eval.eval(model, eval_pos='fake_quant')
@@ -251,6 +252,7 @@ def main(config):
 
 
 if __name__ == '__main__':
+    logger.add(sys.stdout, level='INFO')
     llmc_start_time = time.time()
     parser = argparse.ArgumentParser()
     parser.add_argument('--config', type=str, required=True)

llmc/eval/__init__.py

@@ -2,4 +2,4 @@
 from .eval_code import HumanEval
 from .eval_ppl import PerplexityEval
 from .eval_token_consist import TokenConsistencyEval
-from .eval_vlm import VLMEval
+from .eval_vqa import VQAEval

llmc/eval/eval_base.py

@@ -22,7 +22,8 @@ def __init__(self, tokenizer, config):
             'c4',
             'ptb',
             'custom',
-            'human_eval'
+            'human_eval',
+            'mme',
         ], 'Eval only support wikitext2, c4, ptb, custom, human_eval dataset now.'
         self.seq_len = self.eval_cfg.get('seq_len', None)
         self.bs = self.eval_cfg['bs']