Temporary revert to old vocab conversion for falcon

Author: Galunid

model.py

@@ -563,6 +563,30 @@ def set_gguf_parameters(self):
         self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_epsilon"])
         self.gguf_writer.add_file_type(self.ftype)
 
+    def set_vocab(self):
+        tokens = []
+        scores = []
+        toktypes = []
+
+        from transformers import AutoTokenizer
+        tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
+        vocab_size = self.hparams.get("vocab_size", len(tokenizer.vocab))
+        assert max(tokenizer.vocab.values()) < vocab_size
+
+        reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
+
+        for i in range(vocab_size):
+            tokens.append(reverse_vocab[i])
+            scores.append(0.0) # dummy
+            toktypes.append(gguf.TokenType.NORMAL)
+
+        self.gguf_writer.add_token_list(tokens)
+        self.gguf_writer.add_token_scores(scores)
+        self.gguf_writer.add_token_types(toktypes)
+
+        special_vocab = gguf.SpecialVocab(self.dir_model, load_merges = True, n_vocab = len(tokens))
+        special_vocab.add_to_gguf(self.gguf_writer)
+
     def write_tensors(self):
         block_count = self.hparams.get("num_hidden_layers")
         if block_count is None: