@@ -65,6 +65,7 @@ class Model:
6565 model_name : str | None
6666 metadata_override : Path | None
6767 dir_model_card : Path
68+ remote_hf_model_id : str | None
6869
6970 # subclasses should define this!
7071 model_arch : gguf .MODEL_ARCH
@@ -73,7 +74,7 @@ def __init__(self, dir_model: Path, ftype: gguf.LlamaFileType, fname_out: Path,
7374 use_temp_file : bool = False , eager : bool = False ,
7475 metadata_override : Path | None = None , model_name : str | None = None ,
7576 split_max_tensors : int = 0 , split_max_size : int = 0 , dry_run : bool = False ,
76- small_first_shard : bool = False , hparams : dict [str , Any ] | None = None ):
77+ small_first_shard : bool = False , hparams : dict [str , Any ] | None = None , remote_hf_model_id : str | None = None ):
7778 if type (self ) is Model :
7879 raise TypeError (f"{ type (self ).__name__ !r} )
7980
@@ -83,11 +84,24 @@ def __init__(self, dir_model: Path, ftype: gguf.LlamaFileType, fname_out: Path,
8384 self .is_big_endian = is_big_endian
8485 self .endianess = gguf .GGUFEndian .BIG if is_big_endian else gguf .GGUFEndian .LITTLE
8586 self .use_temp_file = use_temp_file
86- self .lazy = not eager
87- self .part_names = Model .get_model_part_names (self .dir_model , "model" , ".safetensors" )
88- self .is_safetensors = len (self .part_names ) > 0
89- if not self .is_safetensors :
90- self .part_names = Model .get_model_part_names (self .dir_model , "pytorch_model" , ".bin" )
87+ self .lazy = not eager or (remote_hf_model_id is not None )
88+ self .remote_hf_model_id = remote_hf_model_id
89+ if remote_hf_model_id is not None :
90+ self .is_safetensors = True
91+
92+ def get_remote_tensors () -> Iterator [tuple [str , Tensor ]]:
93+ logger .info (f"Using remote model with HuggingFace id: { remote_hf_model_id } )
94+ remote_tensors = gguf .utility .SafetensorRemote .get_list_tensors_hf_model (remote_hf_model_id )
95+ self .tensor_names = set (name for name in remote_tensors .keys ())
96+ for name , remote_tensor in gguf .utility .SafetensorRemote .get_list_tensors_hf_model (remote_hf_model_id ).items ():
97+ yield (name , LazyTorchTensor .from_remote_tensor (remote_tensor ))
98+
99+ self .get_tensors = get_remote_tensors
100+ else :
101+ self .part_names = Model .get_model_part_names (self .dir_model , "model" , ".safetensors" )
102+ self .is_safetensors = len (self .part_names ) > 0
103+ if not self .is_safetensors :
104+ self .part_names = Model .get_model_part_names (self .dir_model , "pytorch_model" , ".bin" )
91105 self .hparams = Model .load_hparams (self .dir_model ) if hparams is None else hparams
92106 self .block_count = self .find_hparam (["n_layers" , "num_hidden_layers" , "n_layer" , "num_layers" ])
93107 self .tensor_map = gguf .get_tensor_name_map (self .model_arch , self .block_count )
@@ -393,6 +407,10 @@ def prepare_metadata(self, vocab_only: bool):
393407
394408 self .metadata = gguf .Metadata .load (self .metadata_override , self .dir_model_card , self .model_name , total_params )
395409
410+ # If we are using HF model id, set the metadata name to the model id
411+ if self .remote_hf_model_id :
412+ self .metadata .name = self .remote_hf_model_id
413+
396414 # Fallback to model directory name if metadata name is still missing
397415 if self .metadata .name is None :
398416 self .metadata .name = self .dir_model .name
@@ -1788,10 +1806,6 @@ def set_gguf_parameters(self):
17881806 self .gguf_writer .add_expert_feed_forward_length (self .hparams ["intermediate_size_moe" ])
17891807
17901808 def modify_tensors (self , data_torch : Tensor , name : str , bid : int | None ):
1791- name = name .replace ("language_model." , "" )
1792- name = name .replace ("feed_forward." , "mlp." ) # a bit hacky for now
1793- name = name .replace (".router.weight" , ".gate.weight" ) # a bit hacky for now
1794-
17951809 # split the gate_up into gate and up
17961810 if "gate_up_proj" in name :
17971811 name_up = name .replace ("gate_up_proj" , "up_proj.weight" )
@@ -5403,6 +5417,14 @@ def from_safetensors_slice(cls, st_slice: Any) -> Tensor:
54035417 lazy = cls (meta = cls .meta_with_dtype_and_shape (dtype , shape ), args = (st_slice ,), func = lambda s : s [:])
54045418 return cast (torch .Tensor , lazy )
54055419
5420+ @classmethod
5421+ def from_remote_tensor (cls , remote_tensor : gguf .utility .RemoteTensor ):
5422+ dtype = cls ._dtype_str_map [remote_tensor .dtype ]
5423+ shape = remote_tensor .shape
5424+ meta = cls .meta_with_dtype_and_shape (dtype , shape )
5425+ lazy = cls (meta = meta , args = (remote_tensor ,), func = lambda r : torch .frombuffer (r .data (), dtype = dtype ).reshape (shape ))
5426+ return cast (torch .Tensor , lazy )
5427+
54065428 @classmethod
54075429 def __torch_function__ (cls , func , types , args = (), kwargs = None ):
54085430 del types # unused
@@ -5480,6 +5502,10 @@ def parse_args() -> argparse.Namespace:
54805502 "--print-supported-models" , action = "store_true" ,
54815503 help = "Print the supported models"
54825504 )
5505+ parser .add_argument (
5506+ "--remote" , action = "store_true" ,
5507+ help = "(Experimental) Read safetensors file remotely without downloading to disk. Config and tokenizer files will still be downloaded. To use this feature, you need to specify Hugging Face model repo name instead of a local directory. For example: 'HuggingFaceTB/SmolLM2-1.7B-Instruct'. Note: To access gated repo, set HF_TOKEN environment variable to your Hugging Face token." ,
5508+ )
54835509
54845510 args = parser .parse_args ()
54855511 if not args .print_supported_models and args .model is None :
@@ -5520,6 +5546,14 @@ def main() -> None:
55205546
55215547 dir_model = args .model
55225548
5549+ if args .remote :
5550+ from huggingface_hub import snapshot_download
5551+ local_dir = snapshot_download (
5552+ repo_id = str (dir_model ),
5553+ allow_patterns = ["LICENSE" , "*.json" , "*.md" , "*.txt" , "tokenizer.model" ])
5554+ dir_model = Path (local_dir )
5555+ logger .info (f"Downloaded config and tokenizer to { local_dir } )
5556+
55235557 if not dir_model .is_dir ():
55245558 logger .error (f'Error: { args .model } )
55255559 sys .exit (1 )
@@ -5541,6 +5575,9 @@ def main() -> None:
55415575
55425576 if args .outfile is not None :
55435577 fname_out = args .outfile
5578+ elif args .remote :
5579+ # if remote, use the model ID as the output file name
5580+ fname_out = Path ("./" + str (args .model ).replace ("/" , "-" ) + "-{ftype}.gguf" )
55445581 else :
55455582 fname_out = dir_model
55465583
@@ -5564,7 +5601,8 @@ def main() -> None:
55645601 metadata_override = args .metadata , model_name = args .model_name ,
55655602 split_max_tensors = args .split_max_tensors ,
55665603 split_max_size = split_str_to_n_bytes (args .split_max_size ), dry_run = args .dry_run ,
5567- small_first_shard = args .no_tensor_first_split )
5604+ small_first_shard = args .no_tensor_first_split ,
5605+ remote_hf_model_id = str (args .model ) if args .remote else None )
55685606
55695607 if args .vocab_only :
55705608 logger .info ("Exporting model vocab..." )
0 commit comments