Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/codecov.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,5 @@ coverage:
project:
default:
target: auto
threshold: 1% # Allow atmost 1% coverage drop from main branch.
threshold: 2% # Allow atmost 2% coverage drop from main branch.
patch: false
6 changes: 3 additions & 3 deletions .github/workflows/example_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ jobs:
uses: ./.github/workflows/_example_tests_runner.yml
secrets: inherit
with:
docker_image: "nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc17"
docker_image: "nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc19"
example: ${{ matrix.example }}
pip_install_extras: "[hf,dev-test]"
runner: linux-amd64-gpu-rtxpro6000-latest-1
Expand All @@ -73,7 +73,7 @@ jobs:
uses: ./.github/workflows/_example_tests_runner.yml
secrets: inherit
with:
docker_image: "nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc17"
docker_image: "nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc19"
example: ${{ matrix.example }}
pip_install_extras: "[hf,dev-test]"
runner: linux-amd64-gpu-rtxpro6000-latest-2
Expand All @@ -85,7 +85,7 @@ jobs:
uses: ./.github/workflows/_example_tests_runner.yml
secrets: inherit
with:
docker_image: "nvcr.io/nvidia/nemo:26.04"
docker_image: "nvcr.io/nvidia/nemo:26.06"
example: megatron_bridge
timeout_minutes: 45
pip_install_extras: "[hf,puzzletron,dev-test]"
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/gpu_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,10 @@ jobs:
container_image: nvcr.io/nvidia/pytorch:26.05-py3
- example: gpu_megatron
timeout: 60
container_image: nvcr.io/nvidia/nemo:26.04
container_image: nvcr.io/nvidia/nemo:26.06
- example: gpu_trtllm
timeout: 15
container_image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc17
container_image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc19
- example: gpu_vllm
timeout: 15
container_image: docker.io/vllm/vllm-openai:v0.20.0
Expand Down
6 changes: 2 additions & 4 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,16 +21,14 @@ repos:
- id: requirements-txt-fixer

- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.12.11
rev: v0.15.20
hooks:
- id: ruff-check
args: [--fix, --exit-non-zero-on-fix]
exclude: ^examples/specdec_bench/specdec_bench/datasets/speed\.py$
- id: ruff-format
exclude: ^examples/specdec_bench/specdec_bench/datasets/speed\.py$

- repo: https://github.com/pre-commit/mirrors-mypy
rev: v1.17.1
rev: v2.1.0
hooks:
- id: mypy

Expand Down
5 changes: 5 additions & 0 deletions docs/source/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,11 @@
exclude_patterns = []
templates_path = ["_templates"]

# Suppress ambiguous cross-reference warnings that arise because EMAConfig and
# QuantizerAttributeConfig both define a field named `type`. Renaming would be
# an API break; silencing the warning here is the least-invasive fix.
suppress_warnings = ["ref.python"]


# -- Options for HTML output -------------------------------------------------

Expand Down
2 changes: 1 addition & 1 deletion examples/dataset/add_nemotron_chat.py
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ def parse_nemotron_conversation(raw_conversations: list) -> list[dict] | None:
if content:
msgs.append({"role": role, "content": content})

return msgs if msgs else None
return msgs or None


async def main(args: argparse.Namespace) -> None:
Expand Down
4 changes: 2 additions & 2 deletions examples/diffusers/cache_diffusion/cache_diffusion/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@

PIXART_DEFAULT_CONFIG = [
{
"wildcard_or_filter_func": lambda name: not re.search(
r"transformer_blocks\.(2[1-7])\.", name
"wildcard_or_filter_func": lambda name: (
not re.search(r"transformer_blocks\.(2[1-7])\.", name)
),
"select_cache_step_func": lambda step: (step % 3) != 0,
}
Expand Down
2 changes: 1 addition & 1 deletion examples/diffusers/quantization/diffusion_trt.py
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,7 @@ def main():
)
args = parser.parse_args()

image_name = args.save_image_as if args.save_image_as else f"{args.model}.png"
image_name = args.save_image_as or f"{args.model}.png"
model_dtype = DTYPE_MAP[args.model]

pipe = PipelineManager.create_pipeline_from(
Expand Down
8 changes: 4 additions & 4 deletions examples/diffusers/quantization/onnx_utils/export.py
Original file line number Diff line number Diff line change
Expand Up @@ -417,9 +417,9 @@ def get_io_shapes(model_id, onnx_load_path, trt_dynamic_shapes):
if onnx_load_path != "":
if model_id in ["sdxl-1.0", "sdxl-turbo"]:
output_name = "latent"
elif model_id in ["sd3-medium"]:
elif model_id == "sd3-medium":
output_name = "sample"
elif model_id in ["sd3.5-medium"]:
elif model_id == "sd3.5-medium":
output_name = "out_hidden_states"
elif model_id in ["flux-dev", "flux-schnell"]:
output_name = "output"
Expand Down Expand Up @@ -499,7 +499,7 @@ def modelopt_export_sd(backbone, onnx_dir, model_name, precision):
if model_name == "flux-dev":
input_names.append("guidance")
output_names = ["latent"]
elif model_name in ["ltx-video-dev"]:
elif model_name == "ltx-video-dev":
input_names = [
"hidden_states",
"encoder_hidden_states",
Expand All @@ -508,7 +508,7 @@ def modelopt_export_sd(backbone, onnx_dir, model_name, precision):
"video_coords",
]
output_names = ["latent"]
elif model_name in ["wan2.2-t2v-14b"]:
elif model_name == "wan2.2-t2v-14b":
input_names = [
"hidden_states",
"timestep",
Expand Down
2 changes: 1 addition & 1 deletion examples/gpt-oss/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@ kernels>=0.9.0,<0.13
trackio<0.21
# transformers>=5.3 avoids CVE-2026-4372 (RCE via the `kernels` Hub download path, which this example installs)
transformers>=5.3
trl>=0.21.0
trl>=1.0
2 changes: 1 addition & 1 deletion examples/gpt-oss/sft.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ def main(script_args, training_args, model_args, quant_args):
# ------------------------
model_kwargs = {
"revision": model_args.model_revision,
"trust_remote_code": model_args.trust_remote_code,
"trust_remote_code": getattr(model_args, "trust_remote_code", False),
"attn_implementation": model_args.attn_implementation,
"dtype": getattr(model_args, "dtype", "bfloat16"),
"use_cache": not training_args.gradient_checkpointing,
Expand Down
1 change: 0 additions & 1 deletion examples/llm_distill/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ This section focuses on demonstrating how to apply Model Optimizer to perform kn
### Docker

For Hugging Face models, please use the PyTorch docker image (e.g., `nvcr.io/nvidia/pytorch:26.01-py3`).
For Megatron-Bridge or Megatron-LM models, use the NeMo container (e.g., `nvcr.io/nvidia/nemo:26.02`) which has all the dependencies installed.
Visit our [installation docs](https://nvidia.github.io/Model-Optimizer/getting_started/2_installation.html) for more information.

Also follow the installation steps below to upgrade to the latest version of Model Optimizer and install example-specific dependencies.
Expand Down
4 changes: 3 additions & 1 deletion examples/llm_distill/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
pyarrow
torchao>=0.14.1
trl>=0.23.0
# TODO: trl>=1.7 stops materializing logits during eval, breaking KD eval loss in
# modelopt/torch/distill/plugins/huggingface.py (_standard_kd_loss). Lift the cap once fixed.
trl>=1.0,<1.7
1 change: 0 additions & 1 deletion examples/llm_ptq/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ This section focuses on Post-training quantization, a technique that reduces mod
### Docker

For Hugging Face models, please use the TensorRT-LLM docker image (e.g., `nvcr.io/nvidia/tensorrt-llm/release:1.2.0`).
For Megatron-Bridge or Megatron-LM models, use the NeMo container (e.g., `nvcr.io/nvidia/nemo:26.02`).
Visit our [installation docs](https://nvidia.github.io/Model-Optimizer/getting_started/2_installation.html) for more information.

Also follow the installation steps below to upgrade to the latest version of Model Optimizer and install example-specific dependencies.
Expand Down
2 changes: 1 addition & 1 deletion examples/llm_qat/dataset_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -539,7 +539,7 @@ def _build_cache_path(
cache_dir: str,
) -> str:
"""Build a deterministic cache path for the blend config."""
base = cache_dir if cache_dir else tempfile.gettempdir()
base = cache_dir or tempfile.gettempdir()

tok_name, tok_fp = _tokenizer_fingerprint(tokenizer)
splits_str = ",".join(f"{k}:{v}" for k, v in sorted(config.splits.items()))
Expand Down
2 changes: 1 addition & 1 deletion examples/llm_qat/notebooks/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
ipywidgets
nvidia-modelopt[all]
trl
trl>=1.0
4 changes: 2 additions & 2 deletions examples/megatron_bridge/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ This directory contains examples of using Model Optimizer with [NeMo Megatron-Br

## Pre-Requisites

Running these examples requires many additional dependencies to be installed (e.g., Megatron-Bridge, Megatron-core, etc.), hence we strongly recommend directly using the NeMo container (e.g., `nvcr.io/nvidia/nemo:26.04`) which has all the dependencies installed.
Running these examples requires many additional dependencies to be installed (e.g., Megatron-Bridge, Megatron-core, etc.), hence we strongly recommend directly using the NeMo container (e.g., `nvcr.io/nvidia/nemo:26.06`) which has all the dependencies installed.

To get the ModelOpt examples scripts, mount your Model-Optimizer repo to the container as follows:

Expand All @@ -30,7 +30,7 @@ if [ ! -d "${MODELOPT_DIR}" ]; then
git clone https://github.com/NVIDIA/Model-Optimizer.git ${MODELOPT_DIR}
fi

export DOCKER_IMAGE=nvcr.io/nvidia/nemo:26.04
export DOCKER_IMAGE=nvcr.io/nvidia/nemo:26.06
docker run \
--gpus all \
--shm-size=16GB \
Expand Down
8 changes: 3 additions & 5 deletions examples/megatron_bridge/prune_minitron.py
Original file line number Diff line number Diff line change
Expand Up @@ -319,11 +319,9 @@ def main(args: argparse.Namespace):
# NAS-based pruning: restrict search space to a smaller set of candidates.
# Allow more choices for MoE FFN as they are generally smaller.
# NOTE: Reduce divisors and increase config['top_k'] to potentially find a better model.
hidden_size_divisor = args.ss_channel_divisor if args.ss_channel_divisor else 256
ffn_hidden_size_divisor = (
args.ss_channel_divisor
if args.ss_channel_divisor
else (256 if (provider.num_moe_experts or 0) > 0 else 512)
hidden_size_divisor = args.ss_channel_divisor or 256
ffn_hidden_size_divisor = args.ss_channel_divisor or (
256 if (provider.num_moe_experts or 0) > 0 else 512
)
ss_config = mtp.mcore_minitron.get_mcore_minitron_config(
hidden_size_divisor=hidden_size_divisor,
Expand Down
2 changes: 1 addition & 1 deletion examples/pruning/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ This section focuses on applying Model Optimizer's state-of-the-art complementar

## Pre-Requisites

For Minitron pruning for Megatron-Bridge / Megatron-LM models, use the NeMo container (e.g., `nvcr.io/nvidia/nemo:26.04`) which has all the dependencies installed.
For Minitron pruning for Megatron-Bridge / Megatron-LM models, use the NeMo container (e.g., `nvcr.io/nvidia/nemo:26.06`) which has all the dependencies installed.

For FastNAS pruning for PyTorch Computer Vision models, no additional dependencies are required.

Expand Down
30 changes: 16 additions & 14 deletions examples/specdec_bench/specdec_bench/datasets/speed.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
# limitations under the License.

# mypy: disable-error-code="index"
# Paper-derived prompt strings carry intentional whitespace/long lines; keep them verbatim.
# ruff: noqa: E501, W291, W293, PLR1704
import random
import re
from enum import Enum
Expand Down Expand Up @@ -74,13 +76,15 @@ class BenchmarkDataset(str, Enum):
BenchmarkDataset.CNN_DAILYMAIL.value: lambda dataset_name, config_name: load_dataset(
dataset_name, config_name, split="test"
),
BenchmarkDataset.HLE.value: lambda dataset_name, config_name: load_dataset(
dataset_name, split="test", revision="021a3d71f516a7ac28ceb8d284969902edf1edeb"
)
if config_name != "train_test_split"
else load_dataset(
dataset_name, split="test", revision="021a3d71f516a7ac28ceb8d284969902edf1edeb"
).train_test_split(test_size=0.5, shuffle=True, seed=42),
BenchmarkDataset.HLE.value: lambda dataset_name, config_name: (
load_dataset(
dataset_name, split="test", revision="021a3d71f516a7ac28ceb8d284969902edf1edeb"
)
if config_name != "train_test_split"
else load_dataset(
dataset_name, split="test", revision="021a3d71f516a7ac28ceb8d284969902edf1edeb"
).train_test_split(test_size=0.5, shuffle=True, seed=42)
),
BenchmarkDataset.LIVECODEBENCH.value: lambda dataset_name, config_name: load_dataset(
"json",
data_files={
Expand Down Expand Up @@ -243,7 +247,7 @@ def _generate_stackselect_prompt(
answers_to_add = (
answers[: answers_to_add_stop + 1]
if answers_to_add_stop >= correct_answer_i
else [answers[correct_answer_i]] + answers[: answers_to_add_stop + 1]
else [answers[correct_answer_i], *answers[: answers_to_add_stop + 1]]
)
random.shuffle(answers_to_add)
for i, answer in enumerate(answers_to_add):
Expand Down Expand Up @@ -368,7 +372,7 @@ def _generate_chatrag_bench_prompt(external_dataset: "Dataset") -> list[Any]:
if message["role"] == "user"
]

return [prompt.format(context=context, question=questions[0])] + questions[1:]
return [prompt.format(context=context, question=questions[0]), *questions[1:]]

@staticmethod
def _generate_coser_prompt(external_dataset: "Dataset") -> str:
Expand Down Expand Up @@ -519,11 +523,9 @@ def _fetch_all_turns_data(
hle_train = hle_train.to_pandas()
hle_train = hle_train[hle_train["image"] == ""]
hle_train["demonstration"] = hle_train.apply(
lambda e: "Question: "
+ e["question"]
+ "\n\nAnswer: "
+ e["rationale"]
+ "\n\n",
lambda e: (
"Question: " + e["question"] + "\n\nAnswer: " + e["rationale"] + "\n\n"
),
axis=1,
)
hle_train["tokens"] = hle_train["demonstration"].apply(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -342,9 +342,7 @@ def main():
parser.add_argument("--output", default=None, help="Path to save JSON results")
args = parser.parse_args()

device = torch.device(
args.device if args.device else ("cuda" if torch.cuda.is_available() else "cpu")
)
device = torch.device(args.device or ("cuda" if torch.cuda.is_available() else "cpu"))
log.info(f"Device: {device}")

weights_path = resolve_weights(args.weights)
Expand Down
5 changes: 1 addition & 4 deletions modelopt/onnx/graph_surgery/encoder_cross_kv.py
Original file line number Diff line number Diff line change
Expand Up @@ -154,10 +154,7 @@ def _add_cross_kv_outputs(
# Update the graph output
for output in list(graph.output):
if output.name == hidden_state_output_name:
dims = [
d.dim_param if d.dim_param else d.dim_value
for d in output.type.tensor_type.shape.dim
]
dims = [d.dim_param or d.dim_value for d in output.type.tensor_type.shape.dim]
new_output = helper.make_tensor_value_info(
encoder_hidden_states_name,
output.type.tensor_type.elem_type,
Expand Down
2 changes: 1 addition & 1 deletion modelopt/onnx/llm_export_utils/quantization_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ def quantize(
f"Only fp8(W8A8), int4_awq(W4A16), nvfp4(W4A4) is supported. You passed an unsupported precision: {precision}."
)

assert lm_head_precision in ["fp16"], (
assert lm_head_precision == "fp16", (
f"Only fp16(unquantized) is supported for lm_head. You passed an unsupported precision: {lm_head_precision}."
)

Expand Down
2 changes: 1 addition & 1 deletion modelopt/onnx/op_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,7 @@ def is_control_flow_op(op_type: str):

def is_multiclass_op(op_type: str):
"""Returns whether the given op type is of Multiclass category or not."""
return op_type in ["Einsum"]
return op_type == "Einsum"


def is_recurrent_op(op_type: str):
Expand Down
2 changes: 1 addition & 1 deletion modelopt/onnx/quantization/graph_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -596,7 +596,7 @@ def build_non_residual_input_map(
non_residual_inputs = {}
no_quantize_inputs = []
for node in graph.nodes:
if node.op in ["Add"]:
if node.op == "Add":
# Add nodes with constant or graph input does not have non-residual input
# Here, A = node.inputs[0], B = node.inputs[1] and A.inputs means producer nodes of A
# TODO: make this check a util?
Expand Down
4 changes: 2 additions & 2 deletions modelopt/onnx/quantization/qdq_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -1490,7 +1490,7 @@ def quantize_weights_to_mxfp8(

# set output type of DQ to FP16
for node in graph.node:
if node.op_type in ["TRT_MXFP8DequantizeLinear"]:
if node.op_type == "TRT_MXFP8DequantizeLinear":
for attr in node.attribute:
if attr.name == "output_dtype":
attr.i = onnx_dtype_map["Half"]
Expand Down Expand Up @@ -1568,7 +1568,7 @@ def _get_precision_dtype() -> str:
logger.debug(f"Found {len(fp4_qdq_nodes)} FP4QDQ nodes to convert")

for node in fp4_qdq_nodes:
idx1 = initializer_indices.get(node.input[0], None)
idx1 = initializer_indices.get(node.input[0])
assert idx1 is not None, f"Initializer for weight '{node.input[0]}' not found."
block_size_attr = next((attr for attr in node.attribute if attr.name == "block_size"), None)
assert block_size_attr is not None, f"block_size attribute not found for {node.name}"
Expand Down
2 changes: 1 addition & 1 deletion modelopt/onnx/trt_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ def get_custom_layers(

for i in range(layer.num_outputs):
output_tensor = layer.get_output(i)
if output_tensor and not all_tensor_info.get(output_tensor.name, None):
if output_tensor and not all_tensor_info.get(output_tensor.name):
all_tensor_info[output_tensor.name] = {
"shape": ["unk" if (s == -1) else s for s in output_tensor.shape],
"dtype": output_tensor.dtype,
Expand Down
2 changes: 1 addition & 1 deletion modelopt/torch/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@

if _Version(_transformers_version) < _Version("4.56") or _Version(
_transformers_version
) >= _Version("5.10"):
) >= _Version("5.13"):
_warnings.warn(
f"transformers {_transformers_version} is not tested with current version of modelopt and may cause issues."
" Please install recommended version with `pip install -U nvidia-modelopt[hf]` if working with HF models.",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ def __del__(self):

def get_engine_bytes(engine: trt.tensorrt.ICudaEngine) -> bytes:
"""Return serialized TensorRT engine bytes."""
return bytearray(engine.serialize()) # type: ignore[return-value]
return bytes(engine.serialize())


def load_engine(buffer: bytes, log_level: int = trt.Logger.ERROR) -> trt.tensorrt.ICudaEngine:
Expand Down
Loading
Loading