Merge pull request #5 from DeepAuto-AI/deepauto/feat/refactor-code

Refactor code in preparation for SGLang PR

Author: daniel-geon-park

.gitignore

@@ -228,5 +228,6 @@ compile_commands.json
 
 1
 
+# Profiling data
 *.nsys-rep
 *.ncu-rep

python/sglang/bench_one_batch.py

@@ -137,6 +137,7 @@ def load_model(server_args, port_args, tp_rank):
         is_embedding=server_args.is_embedding,
         dtype=server_args.dtype,
         quantization=server_args.quantization,
+        is_context_extended=server_args.enable_hip_attention,
     )
     model_runner = ModelRunner(
         model_config=model_config,

python/sglang/srt/configs/model_config.py

@@ -43,6 +43,7 @@ def __init__(
         is_embedding: Optional[bool] = None,
         dtype: str = "auto",
         quantization: Optional[str] = None,
+        is_context_extended: Optional[bool] = None,
     ) -> None:
         self.model_path = model_path
         self.revision = revision
@@ -70,21 +71,20 @@ def __init__(
         derived_context_len = get_context_length(self.hf_text_config)
         if context_length is not None:
             if context_length > derived_context_len:
-                # FIXME: ignore this env flag only when HiP + context extension activated
-                logger.warning(
-                    f"Warning: User-specified context_length ({context_length}) is greater than the derived context_length ({derived_context_len}). "
-                    f"This may lead to incorrect model outputs or CUDA errors."
-                )
-                self.context_len = context_length
-                # if get_bool_env_var("SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN"):
-                # else:
-                #     raise ValueError(
-                #         f"User-specified context_length ({context_length}) is greater than the derived context_length ({derived_context_len}). "
-                #         f"This may lead to incorrect model outputs or CUDA errors. Note that the derived context_length may differ from max_position_embeddings in the model's config. "
-                #         f"To allow overriding this maximum, set the env var SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"
-                #     )
-            else:
-                self.context_len = context_length
+                if is_context_extended:
+                    pass
+                elif get_bool_env_var("SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN"):
+                    logger.warning(
+                        f"Warning: User-specified context_length ({context_length}) is greater than the derived context_length ({derived_context_len}). "
+                        f"This may lead to incorrect model outputs or CUDA errors."
+                    )
+                else:
+                    raise ValueError(
+                        f"User-specified context_length ({context_length}) is greater than the derived context_length ({derived_context_len}). "
+                        f"This may lead to incorrect model outputs or CUDA errors. Note that the derived context_length may differ from max_position_embeddings in the model's config. "
+                        f"To allow overriding this maximum, set the env var SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"
+                    )
+            self.context_len = context_length
         else:
             self.context_len = derived_context_len