Support NemotronH_Omni_Reasoning_V3 in SGLang (#35599)
Signed-off-by: Ryan Stewart <rystewart@nvidia.com> Signed-off-by: rystewart-nvidia <rystewart@nvidia.com> Co-authored-by: elvischenv <219235043+elvischenv@users.noreply.github.com> Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
This commit is contained in:
co-authored by
elvischenv
Po-Han Huang
parent
203d7e812c
commit
fae8cd84cb
@@ -41,6 +41,7 @@ from sglang.srt.model_loader.weight_utils import (
|
||||
)
|
||||
from sglang.srt.models.minimax_m3 import MiniMaxM3SparseForCausalLM
|
||||
from sglang.srt.models.muse_glimmer import MuseGlimmerForConditionalGeneration
|
||||
from sglang.srt.models.nano_nemotron_vl import NemotronH_Omni_Reasoning_V3
|
||||
from sglang.srt.models.utils import WeightsMapper
|
||||
from sglang.srt.utils import get_device
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
@@ -716,6 +717,29 @@ class TestModelOptFp4LoaderSelection(CustomTestCase):
|
||||
|
||||
|
||||
class TestModelOptMixedPrecisionConfig(CustomTestCase):
|
||||
def test_nemotron_h_omni_resolves_fused_qkv_from_split_layers(self):
|
||||
quant_config = ModelOptMixedPrecisionConfig.from_config(
|
||||
{
|
||||
"quant_algo": "MIXED_PRECISION",
|
||||
"quantized_layers": {
|
||||
f"language_model.model.layers.7.mixer.{projection}": {
|
||||
"quant_algo": "FP8"
|
||||
}
|
||||
for projection in ("q_proj", "k_proj", "v_proj")
|
||||
},
|
||||
"packed_modules_mapping": (
|
||||
NemotronH_Omni_Reasoning_V3.packed_modules_mapping
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
quant_config._resolve_quant_algo(
|
||||
"language_model.model.layers.7.mixer.qkv_proj"
|
||||
),
|
||||
"FP8",
|
||||
)
|
||||
|
||||
def test_fp8_pb_wo_dispatches_to_native_block_fp8(self):
|
||||
quant_config = ModelOptMixedPrecisionConfig.from_config(
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user