Moves 2 models from fill-mask task to sentence similarity task

alt_e2eshark/e2e_testing/framework.py

@@ -26,7 +26,7 @@ class ImporterOptions(NamedTuple):
     param_gb_threshold : Optional[float] = None
 
 class CompilerOptions(NamedTuple):
-    """Specify, for specific iree-hal-target-backends, a tuple of extra compiler flags. 
+    """Specify, for specific iree-hal-target-backends, a tuple of extra compiler flags.
        Also allows backend-agnostic options to be included."""
     backend_specific_flags : Dict[str, Tuple[str]] = dict()
     common_extra_args : Tuple[str] = tuple()
@@ -64,6 +64,14 @@ def __init__(
         self.extra_options = ExtraOptions()
         self.update_extra_options()
 
+
+    def update_model_without_ext_data(self):
+        """For large models, which fail opset_version updating, use this method to update without loading external data.
+        This will also trace the graph and copy the external data references which gets wiped out otherwise.
+        """
+        update_no_ext(onnx_model_path=self.model, opset_version=self.opset_version)
+
+
     def forward(self, input: Optional[TestTensors] = None) -> TestTensors:
         """Applies self.model to self.input. Only override if necessary for specific models"""
         input = input.to_numpy().data
@@ -81,7 +89,7 @@ def forward(self, input: Optional[TestTensors] = None) -> TestTensors:
         return TestTensors(model_output)
 
     def update_dim_param_dict(self):
-        """Can be overridden to modify a dictionary of dim parameters (self.dim_param_dict) used to 
+        """Can be overridden to modify a dictionary of dim parameters (self.dim_param_dict) used to
         construct inputs for a model with dynamic dims.
         """
         pass
@@ -119,7 +127,7 @@ def construct_inputs(self) -> TestTensors:
     def apply_postprocessing(self, output: TestTensors):
         """can be overridden to define post-processing methods for individual models"""
         return output
-    
+
     def save_processed_output(self, output: TestTensors, save_to: str, name: str):
         """can be overridden to provide instructions on saving processed outputs (e.g., images, labels, text)"""
         pass
@@ -131,7 +139,7 @@ def get_signature(self, *, from_inputs=True, leave_dynamic=False):
         if not os.path.exists(self.model):
             self.construct_model()
         if not leave_dynamic:
-            self.update_dim_param_dict() 
+            self.update_dim_param_dict()
         return get_signature_for_onnx_model(self.model, from_inputs=from_inputs, dim_param_dict=self.dim_param_dict, leave_dynamic=leave_dynamic)
 
     def load_inputs(self, dir_path):
@@ -154,7 +162,7 @@ def load_golden_outputs(self, dir_path):
         """computes the input signature of the onnx model and loads golden outputs from bin files"""
         shapes, dtypes = self.get_signature(from_inputs=False)
         return TestTensors.load_from(shapes, dtypes, dir_path, "golden_output")
-    
+
     def update_opset_version_and_overwrite(self):
         if not self.opset_version:
             return
@@ -167,7 +175,7 @@ def update_opset_version_and_overwrite(self):
             og_model, self.opset_version
         )
         onnx.save(model, self.model)
-    
+
     def get_metadata(self):
         model_size = os.path.getsize(self.model)
         freq = get_op_frequency(self.model)
@@ -177,7 +185,7 @@ def get_metadata(self):
 
 
 # TODO: extend TestModel to a union, or make TestModel a base class when supporting other frontends
-TestModel = OnnxModelInfo 
+TestModel = OnnxModelInfo
 CompiledArtifact = TypeVar("CompiledArtifact")
 ModelArtifact = Union[Module, onnx.ModelProto]
 CompiledOutput = Union[CompiledArtifact, ort.InferenceSession]

alt_e2eshark/e2e_testing/onnx_utils.py

@@ -215,3 +215,37 @@ def find_node(model: onnx.ModelProto, n: int, op_name: str) -> onnx.NodeProto:
             break
         match_counter += 1
     return key
+
+
+def update_no_ext(onnx_model_path: str, opset_version: int):
+    """Models larger than 2GB do not allow updating opset version, so we update opset version without loading external data. Unfortunately, all external data references get wiped, and we need to manually trace through the graph and copy them back in."""
+    model = onnx.load(onnx_model_path, load_external_data=False)
+    new_model = onnx.version_converter.convert_version(
+        model, opset_version
+    )
+    nv_init_map = {init.name: init for init in model.graph.initializer}
+    for new_init in new_model.graph.initializer:
+        if new_init.name not in nv_init_map.keys():
+            continue
+        old_init = nv_init_map[new_init.name]
+        if old_init.data_location:
+            new_init.Clear()
+            new_init.CopyFrom(old_init)
+    nv_node_map = {node.output[0]: node for node in model.graph.node}
+    for node in new_model.graph.node:
+        name = node.output[0]
+        if name not in nv_node_map.keys():
+            continue
+        og_node = nv_node_map[name]
+        attrs_to_replace = dict()
+        for attr in og_node.attribute:
+            if attr.t.external_data:
+                attrs_to_replace[attr.name] = attr
+        for new_attr in node.attribute:
+            if new_attr.name in attrs_to_replace.keys():
+                old_attr = attrs_to_replace[new_attr.name]
+                new_attr.Clear()
+                new_attr.CopyFrom(old_attr)
+
+    onnx.save(new_model, onnx_model_path)
+

alt_e2eshark/onnx_tests/helper_classes.py

@@ -109,7 +109,6 @@ def find_models(model_dir):
             try:
                 self.export_model()
             except:
-                #print(self.__repr__())
                 raise RuntimeError("Failed to Export class: ", self)
 
             found_models = find_models(model_dir)

alt_e2eshark/onnx_tests/models/external_lists/hf-model-paths/hf-fill-mask-model-list.txt

@@ -47,7 +47,6 @@ pdelobelle/hf_robbert-v2-dutch-base
 facebook/hf_esm2_t6_8M_UR50D
 neuralmind/hf_bert-large-portuguese-cased
 hfl/hf_chinese-roberta-wwm-ext
-sentence-transformers/hf_multi-qa-mpnet-base-cos-v1
 microsoft/hf_codebert-base-mlm
 vinai/hf_phobert-base
 ufal/hf_robeczech-base
@@ -66,7 +65,6 @@ ALINEAR/hf_albert-japanese-v2
 airesearch/hf_wangchanberta-base-att-spm-uncased
 google-bert/hf_bert-base-german-cased
 facebook/hf_esm2_t30_150M_UR50D
-sentence-transformers/hf_all-mpnet-base-v1
 ai4bharat/hf_IndicBERTv2-MLM-only
 dbmdz/hf_bert-base-italian-xxl-cased
 cahya/hf_bert-base-indonesian-1.5G

alt_e2eshark/onnx_tests/models/external_lists/hf-model-paths/hf-multiple-choice-model-list.txt

@@ -12,7 +12,6 @@ irfanamal/hf_distilbert_multiple_choice
 persiannlp/hf_parsbert-base-parsinlu-multiple-choice
 irfanamal/hf_electra_multiple_choice
 VuongQuoc/hf_checkpoints_28_9_microsoft_deberta_V2
-Liberty-L/hf_multiple_choice_model
 VuongQuoc/hf_checkpoints_26_9_microsoft_deberta_21_9
 VuongQuoc/hf_checkpoints_29_9_microsoft_deberta_V1
 irfanamal/hf_bert_science_multiple_choice
@@ -45,13 +44,13 @@ sledz08/hf_finetuned-bert-piqa
 irfanamal/hf_distilbert_science_multiple_choice
 VuongQuoc/hf_checkpoints_28_9_microsoft_deberta_V5
 PaulTran/hf_phobert-base-finetuned
-Liberty-L/hf_Multiple_Choice_EN
 pallie/hf_mcQA_model_bert
-Liberty-L/hf_Multiple_Choice
 paulovsantanas/hf_llm-mdeberta-v3-swag
 Xenopilus/hf_electra-base-multiple-choice-v2
 edwardyeung04/hf_deberta-v3-large_test_9e-6
 reichenbach/hf_bert_base_swag_model
 sontn122/hf_content
 VuongQuoc/hf_checkpoints_28_9_microsoft_deberta_V4
 vuminhtue/hf_LLM
+Liberty-L/hf_Multiple_Choice_swag
+Liberty-L/hf_Multiple_Choice_swag_lr

alt_e2eshark/onnx_tests/models/external_lists/hf-model-paths/hf-sentence-similarity-model-list.txt

@@ -0,0 +1,2 @@
+sentence-transformers/hf_all-mpnet-base-v1
+sentence-transformers/hf_multi-qa-mpnet-base-cos-v1

alt_e2eshark/onnx_tests/models/external_lists/hf-model-paths/hf-text-generation-model-list.txt

@@ -19,7 +19,6 @@ TheBloke/hf_Mistral-7B-Instruct-v0.2-GPTQ
 unsloth/hf_Meta-Llama-3.1-8B-Instruct-bnb-4bit
 petals-team/hf_StableBeluga2
 OpenAssistant/hf_oasst-sft-4-pythia-12b-epoch-3.5
-wasmdashai/hf_wasmai-7b-v1
 datificate/hf_gpt2-small-spanish
 microsoft/hf_Phi-3.5-mini-instruct
 cecibas/hf_Midnight-Miqu-70B-v1.5-4bit

alt_e2eshark/onnx_tests/models/external_lists/hf-model-shards/hf-fill-mask-shard.txt

@@ -47,7 +47,6 @@ hf_robbert-v2-dutch-base
 hf_esm2_t6_8M_UR50D
 hf_bert-large-portuguese-cased
 hf_chinese-roberta-wwm-ext
-hf_multi-qa-mpnet-base-cos-v1
 hf_codebert-base-mlm
 hf_phobert-base
 hf_robeczech-base
@@ -66,7 +65,6 @@ hf_albert-japanese-v2
 hf_wangchanberta-base-att-spm-uncased
 hf_bert-base-german-cased
 hf_esm2_t30_150M_UR50D
-hf_all-mpnet-base-v1
 hf_IndicBERTv2-MLM-only
 hf_bert-base-italian-xxl-cased
 hf_bert-base-indonesian-1.5G

alt_e2eshark/onnx_tests/models/external_lists/hf-model-shards/hf-multiple-choice-shard.txt

@@ -12,7 +12,6 @@ hf_distilbert_multiple_choice
 hf_parsbert-base-parsinlu-multiple-choice
 hf_electra_multiple_choice
 hf_checkpoints_28_9_microsoft_deberta_V2
-hf_multiple_choice_model
 hf_checkpoints_26_9_microsoft_deberta_21_9
 hf_checkpoints_29_9_microsoft_deberta_V1
 hf_bert_science_multiple_choice
@@ -45,13 +44,13 @@ hf_finetuned-bert-piqa
 hf_distilbert_science_multiple_choice
 hf_checkpoints_28_9_microsoft_deberta_V5
 hf_phobert-base-finetuned
-hf_Multiple_Choice_EN
 hf_mcQA_model_bert
-hf_Multiple_Choice
 hf_llm-mdeberta-v3-swag
 hf_electra-base-multiple-choice-v2
 hf_deberta-v3-large_test_9e-6
 hf_bert_base_swag_model
 hf_content
 hf_checkpoints_28_9_microsoft_deberta_V4
 hf_LLM
+hf_Multiple_Choice_swag
+hf_Multiple_Choice_swag_lr

alt_e2eshark/onnx_tests/models/external_lists/hf-model-shards/hf-sentence-similarity-shard.txt

@@ -0,0 +1,2 @@
+hf_all-mpnet-base-v1
+hf_multi-qa-mpnet-base-cos-v1

alt_e2eshark/onnx_tests/models/external_lists/hf-model-shards/hf-text-generation-shard.txt

@@ -19,7 +19,6 @@ hf_Mistral-7B-Instruct-v0.2-GPTQ
 hf_Meta-Llama-3.1-8B-Instruct-bnb-4bit
 hf_StableBeluga2
 hf_oasst-sft-4-pythia-12b-epoch-3.5
-hf_wasmai-7b-v1
 hf_gpt2-small-spanish
 hf_Phi-3.5-mini-instruct
 hf_Midnight-Miqu-70B-v1.5-4bit