Use tokenizer.save_pretrained api in base_blockwise_sparsification.py

ModelTC · Jan 11, 2025 · 371fbab · 371fbab
1 parent 51a26eb
commit 371fbab
Showing 1 changed file with 1 addition and 2 deletions.
diff --git a/llmc/compression/sparsification/base_blockwise_sparsification.py b/llmc/compression/sparsification/base_blockwise_sparsification.py
@@ -118,8 +118,7 @@ def deploy(self, deploy_format):
 
     @torch.no_grad()
     def copy_tokenizer(self, path):
-        for substring in self.config.save.get('tokenizer_file_substring', ['token']):
-            copy_files(self.config.model.path, path, substring)
+        self.model.tokenizer.save_pretrained(path)
         logger.info('copy tokenizer done --')
 
     @torch.no_grad()