Revert "Temporary revert to old vocab conversion for falcon"

This reverts commit 63dd07a.

Author: Galunid

model.py

@@ -568,30 +568,6 @@ def set_gguf_parameters(self):
         self.gguf_writer.add_layer_norm_eps(self.hparams["layer_norm_epsilon"])
         self.gguf_writer.add_file_type(self.ftype)
 
-    def set_vocab(self):
-        tokens = []
-        scores = []
-        toktypes = []
-
-        from transformers import AutoTokenizer
-        tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
-        vocab_size = self.hparams.get("vocab_size", len(tokenizer.vocab))
-        assert max(tokenizer.vocab.values()) < vocab_size
-
-        reverse_vocab = {id: encoded_tok for encoded_tok, id in tokenizer.vocab.items()}
-
-        for i in range(vocab_size):
-            tokens.append(reverse_vocab[i])
-            scores.append(0.0) # dummy
-            toktypes.append(gguf.TokenType.NORMAL)
-
-        self.gguf_writer.add_token_list(tokens)
-        self.gguf_writer.add_token_scores(scores)
-        self.gguf_writer.add_token_types(toktypes)
-
-        special_vocab = gguf.SpecialVocab(self.dir_model, load_merges = True, n_vocab = len(tokens))
-        special_vocab.add_to_gguf(self.gguf_writer)
-
     def write_tensors(self):
         block_count = self.hparams.get("num_hidden_layers")
         if block_count is None: