Skip to content

Commit

Permalink
Update OGA LEAP recipes (#289)
Browse files Browse the repository at this point in the history
  • Loading branch information
jeremyfowers authored Feb 6, 2025
1 parent d43e50f commit b805839
Show file tree
Hide file tree
Showing 2 changed files with 13 additions and 4 deletions.
15 changes: 12 additions & 3 deletions src/lemonade/leap.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,9 @@ def from_pretrained(
Recipe choices:
- hf-cpu: Huggingface Transformers implementation for CPU with max-perf settings
- hf-dgpu: Huggingface Transformers implementation on dGPU (via device="cuda")
- oga-dml: DirectML implementation for iGPU based on onnxruntime-genai
- oga-cpu: CPU implementation based on onnxruntime-genai
- oga-dml: DirectML implementation for iGPU based on onnxruntime-genai-directml
- oga-hybird: AMD Ryzen AI Hybrid implementation based on onnxruntime-genai
Returns:
- model: LLM instance with a generate() method that invokes the recipe
Expand Down Expand Up @@ -89,21 +91,28 @@ def from_pretrained(

# Make sure the user chose a supported runtime, e.g., oga-cpu
user_backend = recipe.split("oga-")[1]
supported_backends = ["cpu", "igpu", "npu", "hybrid", "cuda"]
supported_backends = ["cpu", "igpu", "npu", "hybrid"]
supported_recipes = [f"oga-{backend}" for backend in supported_backends]
if recipe not in supported_recipes:
raise NotSupported(
"Selected OGA recipe is not supported. "
f"The supported OGA recipes are: {supported_recipes}"
)

backend_to_dtype = {
"cpu": "fp32",
"igpu": "fp16",
"hybrid": "int4",
"npu": "int4",
}

state = _make_state(recipe, checkpoint)

state = oga.OgaLoad().run(
state,
input=checkpoint,
device=user_backend,
dtype="int4",
dtype=backend_to_dtype[user_backend],
)

return state.model, state.tokenizer
Expand Down
2 changes: 1 addition & 1 deletion src/turnkeyml/version.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = "5.1.0"
__version__ = "5.1.1"

0 comments on commit b805839

Please sign in to comment.