Support NemotronH_Omni_Reasoning_V3 in SGLang (#35599)

Signed-off-by: Ryan Stewart <rystewart@nvidia.com>
Signed-off-by: rystewart-nvidia <rystewart@nvidia.com>
Co-authored-by: elvischenv <219235043+elvischenv@users.noreply.github.com>
Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
This commit is contained in:
rystewart-nvidia
2026-09-10 16:57:22 -07:00
committed by GitHub
co-authored by elvischenv Po-Han Huang
parent 203d7e812c
commit fae8cd84cb
20 changed files with 1080 additions and 62 deletions
@@ -41,6 +41,7 @@ from sglang.srt.model_loader.weight_utils import (
)
from sglang.srt.models.minimax_m3 import MiniMaxM3SparseForCausalLM
from sglang.srt.models.muse_glimmer import MuseGlimmerForConditionalGeneration
from sglang.srt.models.nano_nemotron_vl import NemotronH_Omni_Reasoning_V3
from sglang.srt.models.utils import WeightsMapper
from sglang.srt.utils import get_device
from sglang.test.ci.ci_register import register_cuda_ci
@@ -716,6 +717,29 @@ class TestModelOptFp4LoaderSelection(CustomTestCase):
class TestModelOptMixedPrecisionConfig(CustomTestCase):
def test_nemotron_h_omni_resolves_fused_qkv_from_split_layers(self):
quant_config = ModelOptMixedPrecisionConfig.from_config(
{
"quant_algo": "MIXED_PRECISION",
"quantized_layers": {
f"language_model.model.layers.7.mixer.{projection}": {
"quant_algo": "FP8"
}
for projection in ("q_proj", "k_proj", "v_proj")
},
"packed_modules_mapping": (
NemotronH_Omni_Reasoning_V3.packed_modules_mapping
),
}
)
self.assertEqual(
quant_config._resolve_quant_algo(
"language_model.model.layers.7.mixer.qkv_proj"
),
"FP8",
)
def test_fp8_pb_wo_dispatches_to_native_block_fp8(self):
quant_config = ModelOptMixedPrecisionConfig.from_config(
{