From 6fe5596ebb1bdc17fa55104e9f87a3cfd77566b0 Mon Sep 17 00:00:00 2001 From: elvischenv <219235043+elvischenv@users.noreply.github.com> Date: Thu, 15 Oct 2026 21:50:09 +0800 Subject: [PATCH] Support EPLB for Nemotron-H (#43685) Co-authored-by: Claude Opus 5.5 --- .../arg_groups/model_overrides/nemotron_h.py | 51 ++++++++++ python/sglang/srt/models/nano_nemotron_vl.py | 10 ++ python/sglang/srt/models/nemotron_h.py | 96 +++++++++++++++++-- python/sglang/srt/models/nemotron_h_mtp.py | 15 +-- .../test_nvidia_nemotron_3_super_nvfp4.py | 48 +++++++++- .../unit/models/test_nano_nemotron_vl.py | 31 ++++++ test/registered/unit/test_model_overrides.py | 81 ++++++++++++++++ 7 files changed, 312 insertions(+), 20 deletions(-) diff --git a/python/sglang/srt/arg_groups/model_overrides/nemotron_h.py b/python/sglang/srt/arg_groups/model_overrides/nemotron_h.py index d5cb122fc4..2fcac83fb4 100644 --- a/python/sglang/srt/arg_groups/model_overrides/nemotron_h.py +++ b/python/sglang/srt/arg_groups/model_overrides/nemotron_h.py @@ -54,6 +54,14 @@ def _nemotron_h_overrides(server_args: Any, hf_config: Any) -> dict: quantization = model_config.quantization overrides["quantization"] = quantization + uses_expert_remap = ( + cfg.enable_eplb + or cfg.ep_num_redundant_experts > 0 + or cfg.init_expert_location != "trivial" + ) + if uses_expert_remap: + _validate_expert_remap(cfg, hf_config, model_arch) + has_w4a16_moe_layers = False if is_modelopt and quantization == "modelopt_mixed": has_w4a16_moe_layers = any( @@ -76,6 +84,18 @@ def _nemotron_h_overrides(server_args: Any, hf_config: Any) -> dict: "Use marlin as MoE runner backend for " f"{model_arch} with W4A16_NVFP4 MoE layers" ) + elif ( + uses_expert_remap + and cfg.moe_runner_backend == "auto" + and get_platform().is_sm100 + and cfg.moe_a2a_backend == "none" + ): + # flashinfer_trtllm routes inside the kernel and cannot take the remap. + overrides["moe_runner_backend"] = "flashinfer_trtllm_routed" + logger.info( + "Use flashinfer_trtllm_routed as MoE runner backend on sm100 for " + f"{model_arch} with EPLB or redundant experts" + ) elif (is_modelopt or model_config.quantization is None) and ( cfg.moe_runner_backend == "auto" ): @@ -127,3 +147,34 @@ def _nemotron_h_overrides(server_args: Any, hf_config: Any) -> dict: elif get_platform().is_sm100: overrides["attention_backend"] = "trtllm_mha" return overrides + + +def _validate_expert_remap(cfg: Any, hf_config: Any, model_arch: str) -> None: + if cfg.moe_runner_backend == "flashinfer_trtllm": + raise ValueError( + "flashinfer_trtllm routes inside the kernel and does not support EPLB or " + "redundant experts on Nemotron-H; use " + "--moe-runner-backend flashinfer_trtllm_routed." + ) + if ( + cfg.enable_eplb + and cfg.moe_runner_backend == "flashinfer_cutedsl" + and cfg.moe_a2a_backend == "none" + ): + raise ValueError( + "flashinfer_cutedsl without an a2a backend does not migrate its expert " + "scales on EPLB rebalance; use --moe-runner-backend " + "flashinfer_trtllm_routed or set --moe-a2a-backend." + ) + if model_arch == "NemotronHPuzzleForCausalLM": + num_routed_experts = { + block["n_routed_experts"] + for block in hf_config.block_configs + if block["block_type"] == "moe" + } + if len(num_routed_experts) > 1: + raise ValueError( + "EPLB and redundant experts need the same routed expert count on " + "every MoE layer, got " + f"{sorted(num_routed_experts)}." + ) diff --git a/python/sglang/srt/models/nano_nemotron_vl.py b/python/sglang/srt/models/nano_nemotron_vl.py index b8c87afe8c..833ce47daa 100644 --- a/python/sglang/srt/models/nano_nemotron_vl.py +++ b/python/sglang/srt/models/nano_nemotron_vl.py @@ -128,6 +128,16 @@ class NemotronH_Nano_VL_V2(EVS): self.config = config + @property + def routed_experts_weights_of_layer(self): + return self.language_model.routed_experts_weights_of_layer + + @classmethod + def get_model_config_for_expert_location(cls, config): + return NemotronHForCausalLM.get_model_config_for_expert_location( + config.llm_config + ) + def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array: im_start_id: int = mm_inputs.im_start_id im_end_id: int = mm_inputs.im_end_id diff --git a/python/sglang/srt/models/nemotron_h.py b/python/sglang/srt/models/nemotron_h.py index a8b48916d2..c32b608f0a 100644 --- a/python/sglang/srt/models/nemotron_h.py +++ b/python/sglang/srt/models/nemotron_h.py @@ -26,6 +26,9 @@ from torch import nn from sglang.srt.compilation.compilation_config import register_split_op from sglang.srt.configs import NemotronHConfig from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MLP, MOE +from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder +from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation +from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import ReLU2 from sglang.srt.layers.attention.hybrid_linear_attn_backend import ( HybridLinearAttnBackend, @@ -50,6 +53,7 @@ from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.moe.topk import TopK from sglang.srt.layers.moe.utils import ( RoutingMethodType, + filter_moe_weight_param_global_expert, get_moe_a2a_backend, ) from sglang.srt.layers.quantization import QuantizationConfig @@ -82,6 +86,7 @@ from sglang.srt.models.nemotron_h_utils import make_stage_boundary, stage_facts from sglang.srt.models.utils import WeightsMapper from sglang.srt.runtime_context import get_exec, get_parallel from sglang.srt.utils import ( + LazyValue, add_prefix, get_current_device_stream_fast, is_cuda, @@ -170,9 +175,12 @@ class NemotronHMoE(nn.Module): layer_idx: int, quant_config: QuantizationConfig | None = None, prefix: str = "", + is_nextn: bool = False, ) -> None: super().__init__() + self.layer_id = layer_idx + self.is_nextn = is_nextn self.routed_scaling_factor = config.routed_scaling_factor self.device_module = torch.get_device_module() @@ -279,16 +287,20 @@ class NemotronHMoE(nn.Module): def _forward_core( self, hidden_states: torch.Tensor, + num_token_non_padded: torch.Tensor | None, ) -> tuple[torch.Tensor, torch.Tensor | None]: if _is_cuda and ( not get_moe_a2a_backend().is_flashinfer() or get_is_capture_mode() ): - return self._forward_core_shared_routed_overlap(hidden_states) - return self._forward_core_normal(hidden_states) + return self._forward_core_shared_routed_overlap( + hidden_states, num_token_non_padded + ) + return self._forward_core_normal(hidden_states, num_token_non_padded) def _forward_core_normal( self, hidden_states: torch.Tensor, + num_token_non_padded: torch.Tensor | None, ) -> tuple[torch.Tensor, torch.Tensor | None]: # router_scores: [num_tokens, num_experts] # bf16 gemm on tensor cores with fp32 accumulation/output for sigmoid/topk. @@ -299,7 +311,16 @@ class NemotronHMoE(nn.Module): shared_output = self.shared_experts(hidden_states) else: shared_output = None - topk_output = self.topk(hidden_states, router_logits) + topk_output = self.topk( + hidden_states, + router_logits, + num_token_non_padded=num_token_non_padded, + expert_location_dispatch_info=( + ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id) + if not self.is_nextn + else None + ), + ) if self.use_latent_moe: hidden_states = self._apply_fc1_latent_proj(hidden_states) final_hidden_states = self.experts(hidden_states, topk_output) @@ -308,6 +329,7 @@ class NemotronHMoE(nn.Module): def _forward_core_shared_routed_overlap( self, hidden_states: torch.Tensor, + num_token_non_padded: torch.Tensor | None, ) -> tuple[torch.Tensor, torch.Tensor | None]: alt_stream = _get_or_create_alt_stream(self.device_module) @@ -324,7 +346,16 @@ class NemotronHMoE(nn.Module): router_logits = torch.mm( hidden_states, self.gate.weight.t(), out_dtype=torch.float32 ) - topk_output = self.topk(hidden_states, router_logits) + topk_output = self.topk( + hidden_states, + router_logits, + num_token_non_padded=num_token_non_padded, + expert_location_dispatch_info=( + ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id) + if not self.is_nextn + else None + ), + ) if self.use_latent_moe: hidden_states = self._apply_fc1_latent_proj(hidden_states) final_hidden_states = self.experts(hidden_states, topk_output) @@ -332,6 +363,16 @@ class NemotronHMoE(nn.Module): return final_hidden_states, shared_output + def get_moe_weights(self) -> list[torch.Tensor]: + return [ + x.data + for name, x in self.experts.named_parameters() + if name not in ["correction_bias"] + and filter_moe_weight_param_global_expert( + name, x, self.experts.num_local_experts + ) + ] + def _apply_latent_projection( self, final_hidden_states: torch.Tensor, @@ -351,11 +392,19 @@ class NemotronHMoE(nn.Module): def forward( self, hidden_states: torch.Tensor, + forward_batch: ForwardBatch | None = None, ) -> torch.Tensor: num_tokens, hidden_dim = hidden_states.shape # routed_scaling_factor is fused into the experts call (applied by the # MoE runner / topk), so final_hidden_states is already scaled. - final_hidden_states, shared_output = self._forward_core(hidden_states) + final_hidden_states, shared_output = self._forward_core( + hidden_states, + num_token_non_padded=( + forward_batch.moe_num_token_non_padded() + if forward_batch is not None + else None + ), + ) if self.use_latent_moe: final_hidden_states = self._apply_latent_projection( @@ -430,6 +479,7 @@ class NemotronHMoEDecoderLayer(NemotronHMLPLikeDecoderLayer): layer_idx: int, quant_config: QuantizationConfig | None = None, prefix: str = "", + is_nextn: bool = False, ) -> None: super().__init__() layer_config = config.get_nemotron_h_config_for_layer(layer_idx) @@ -440,11 +490,22 @@ class NemotronHMoEDecoderLayer(NemotronHMLPLikeDecoderLayer): layer_idx=layer_idx, quant_config=quant_config, prefix=f"{prefix}.mixer", + is_nextn=is_nextn, ) self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon) self._init_stage_boundary(config, layer_idx) + def forward( + self, + *, + hidden_states, + forward_batch: ForwardBatch, + ): + hidden_states = self.boundary.prepare(hidden_states, forward_batch) + hidden_states = self.mixer.forward(hidden_states, forward_batch=forward_batch) + return self.boundary.finish(hidden_states, forward_batch) + class NemotronHAttnLikeDecoderLayer(nn.Module): """A mixer (Mamba / attention) stage. Before an FFN stage it leaves its @@ -729,10 +790,11 @@ class NemotronHModel(nn.Module): layer = self.layers[i] if not isinstance(layer, Layers): raise ValueError(f"Unknown layer type: {type(layer)}") - hidden_states = layer.forward( - hidden_states=hidden_states, - forward_batch=forward_batch, - ) + with get_global_expert_distribution_recorder().with_current_layer(i): + hidden_states = layer.forward( + hidden_states=hidden_states, + forward_batch=forward_batch, + ) if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) @@ -842,6 +904,22 @@ class NemotronHForCausalLM(nn.Module): self.logits_processor = LogitsProcessor(config) self.capture_aux_hidden_states = False + self.routed_experts_weights_of_layer = LazyValue( + lambda: { + layer_id: self.model.layers[layer_id].mixer.get_moe_weights() + for layer_id in range(self.model.start_layer, self.model.end_layer) + if isinstance(self.model.layers[layer_id], NemotronHMoEDecoderLayer) + } + ) + + @classmethod + def get_model_config_for_expert_location(cls, config): + return ModelConfigForExpertLocation( + num_layers=config.num_hidden_layers, + num_logical_experts=config.max_n_routed_experts, + num_groups=config.n_group, + ) + def _init_model( self, config: NemotronHConfig, diff --git a/python/sglang/srt/models/nemotron_h_mtp.py b/python/sglang/srt/models/nemotron_h_mtp.py index 630fbd43ff..2fbed71bbf 100644 --- a/python/sglang/srt/models/nemotron_h_mtp.py +++ b/python/sglang/srt/models/nemotron_h_mtp.py @@ -18,6 +18,7 @@ import torch from torch import nn from sglang.srt.configs import NemotronHConfig +from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.layer_boundary import layer_stack from sglang.srt.layers.layer_boundary.residual import batch as residual_batch @@ -137,6 +138,7 @@ class NemotronHMTPMoEDecoderLayer(NemotronHMoEDecoderLayer): layer_idx=layer_idx, quant_config=quant_config, prefix=prefix, + is_nextn=True, ) self.has_start_projections = has_start_projections self.has_end_norm = has_end_norm @@ -302,12 +304,13 @@ class NemotronHMultiTokenPredictor(nn.Module): hidden_states = forward_batch.spec_info.hidden_states residual_batch.start(forward_batch) - for i in range(self.pattern_len): - hidden_states = self.layers[str(i)]( - inputs_embeds=inputs_embeds, - hidden_states=hidden_states, - forward_batch=forward_batch, - ) + with get_global_expert_distribution_recorder().disable_this_region(): + for i in range(self.pattern_len): + hidden_states = self.layers[str(i)]( + inputs_embeds=inputs_embeds, + hidden_states=hidden_states, + forward_batch=forward_batch, + ) return residual_batch.take_output(hidden_states, forward_batch) diff --git a/test/registered/e2e/models_large/test_nvidia_nemotron_3_super_nvfp4.py b/test/registered/e2e/models_large/test_nvidia_nemotron_3_super_nvfp4.py index ae7e96f2d1..4561e2f955 100644 --- a/test/registered/e2e/models_large/test_nvidia_nemotron_3_super_nvfp4.py +++ b/test/registered/e2e/models_large/test_nvidia_nemotron_3_super_nvfp4.py @@ -2,7 +2,6 @@ import unittest from types import SimpleNamespace from sglang.srt.environ import envs -from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( @@ -10,6 +9,7 @@ from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, CustomTestCase, popen_launch_server, + terminate_and_kill_process_tree, ) register_cuda_ci(est_time=1600, stage="nightly", runner_config="4-gpu-b200") @@ -53,6 +53,16 @@ DP_ATTENTION_EP_ARGS = [ "8192", ] +EPLB_ARGS = [ + "--ep-size", + "4", + "--enable-eplb", + "--ep-num-redundant-experts", + "32", + "--eplb-rebalance-num-iterations", + "50", +] + MTP_ARGS = [ "--speculative-algorithm", "EAGLE", @@ -104,7 +114,8 @@ class TestNvidiaNemotron3SuperNVFP4(CustomTestCase): @classmethod def tearDownClass(cls): - kill_process_tree(cls.process.pid) + if hasattr(cls, "process") and cls.process: + terminate_and_kill_process_tree(cls.process) def test_gsm8k(self): _run_gsm8k(self) @@ -131,7 +142,8 @@ class TestNvidiaNemotron3SuperNVFP4PerToken(CustomTestCase): @classmethod def tearDownClass(cls): - kill_process_tree(cls.process.pid) + if hasattr(cls, "process") and cls.process: + terminate_and_kill_process_tree(cls.process) def test_gsm8k(self): _run_gsm8k(self, threshold=0.93) @@ -152,7 +164,32 @@ class TestNvidiaNemotron3SuperNVFP4MTP(CustomTestCase): @classmethod def tearDownClass(cls): - kill_process_tree(cls.process.pid) + if hasattr(cls, "process") and cls.process: + terminate_and_kill_process_tree(cls.process) + + def test_gsm8k(self): + _run_gsm8k(self) + + +class TestNvidiaNemotron3SuperNVFP4EPLB(CustomTestCase): + """EP with dynamic EPLB rebalancing during the eval.""" + + @classmethod + def setUpClass(cls): + cls.model = NEMOTRON_3_SUPER_NVFP4_MODEL + cls.base_url = DEFAULT_URL_FOR_TEST + with envs.SGLANG_ENABLE_ASYNC_ASSERT.override(0): + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=NEMOTRON_3_SUPER_NVFP4_ARGS + EPLB_ARGS, + ) + + @classmethod + def tearDownClass(cls): + if hasattr(cls, "process") and cls.process: + terminate_and_kill_process_tree(cls.process) def test_gsm8k(self): _run_gsm8k(self) @@ -179,7 +216,8 @@ class TestNvidiaNemotron3SuperNVFP4DPAttentionEP(CustomTestCase): @classmethod def tearDownClass(cls): - kill_process_tree(cls.process.pid) + if hasattr(cls, "process") and cls.process: + terminate_and_kill_process_tree(cls.process) def test_gsm8k(self): _run_gsm8k(self) diff --git a/test/registered/unit/models/test_nano_nemotron_vl.py b/test/registered/unit/models/test_nano_nemotron_vl.py index 63782c7736..3c422f48cc 100644 --- a/test/registered/unit/models/test_nano_nemotron_vl.py +++ b/test/registered/unit/models/test_nano_nemotron_vl.py @@ -8,6 +8,8 @@ from types import SimpleNamespace import torch import torch.nn as nn +from sglang.srt.configs import NemotronHConfig +from sglang.srt.configs.nemotron_h import MOE from sglang.srt.managers.schedule_batch import ( Modality, MultimodalDataItem, @@ -281,6 +283,35 @@ class TestNemotronHOmniModel(CustomTestCase): ["sound_encoder.projection.weight"], ) + def test_expert_location_config_spans_full_hybrid_stack(self): + """The VL wrapper exposes the full-stack expert-location config for EPLB.""" + pattern = "ME*E-E" + llm_config = NemotronHConfig( + hybrid_override_pattern=pattern, n_routed_experts=8, n_group=1 + ) + + expert_config = ( + NemotronH_Omni_Reasoning_V3.get_model_config_for_expert_location( + SimpleNamespace(llm_config=llm_config) + ) + ) + + self.assertEqual(expert_config.num_layers, len(pattern)) + moe_layer_ids = [i for i, c in enumerate(pattern) if c == MOE] + self.assertLess(max(moe_layer_ids), expert_config.num_layers) + self.assertEqual(expert_config.num_logical_experts, 8) + + def test_routed_expert_weights_come_from_language_model(self): + """The VL wrapper exposes the language model's expert weights for EPLB.""" + model = object.__new__(NemotronH_Omni_Reasoning_V3) + nn.Module.__init__(model) + weights_of_layer = {1: [torch.ones(1)]} + model.language_model = SimpleNamespace( + routed_experts_weights_of_layer=weights_of_layer + ) + + self.assertIs(model.routed_experts_weights_of_layer, weights_of_layer) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/test_model_overrides.py b/test/registered/unit/test_model_overrides.py index 3324b722ed..2f960749a3 100644 --- a/test/registered/unit/test_model_overrides.py +++ b/test/registered/unit/test_model_overrides.py @@ -1290,6 +1290,9 @@ class TestGoldenModelOverrides(_IsolatedPublish): speculative_draft_attention_backend=None, page_size=None, mamba_radix_cache_strategy="auto", + enable_eplb=False, + ep_num_redundant_experts=0, + init_expert_location="trivial", _model_config=model_config, ), hf_config, @@ -1359,6 +1362,78 @@ class TestGoldenModelOverrides(_IsolatedPublish): }, ) + def test_nemotron_h_eplb_picks_routed_runner_on_sm100(self): + """flashinfer_trtllm cannot take the EPLB remap; with redundant experts it + fails its routing shape check at startup.""" + from sglang.srt.arg_groups.model_overrides.nemotron_h import ( + _nemotron_h_overrides, + ) + + for quantization in ("modelopt_fp4", "nvfp4_online"): + with self.subTest(quantization): + server_args, hf_config = self._nemotron_h_args(quantized_layers={}) + server_args.enable_eplb = True + server_args.ep_num_redundant_experts = 32 + if quantization == "nvfp4_online": + server_args.quantization = quantization + server_args._model_config.quantization = quantization + + with ( + override_platform(is_sm100=True), + override_platform(is_blackwell=True), + ): + overrides = _nemotron_h_overrides(server_args, hf_config) + + self.assertEqual( + overrides["moe_runner_backend"], "flashinfer_trtllm_routed" + ) + + def test_nemotron_h_expert_remap_validation(self): + from sglang.srt.arg_groups.model_overrides.nemotron_h import ( + _validate_expert_remap, + ) + + def puzzle(*n_routed_experts): + return SimpleNamespace( + block_configs=[ + {"block_type": "moe", "n_routed_experts": n} + for n in n_routed_experts + ] + + [{"block_type": "mamba"}] + ) + + def args(moe_runner_backend, moe_a2a_backend="none"): + return SimpleNamespace( + enable_eplb=True, + moe_runner_backend=moe_runner_backend, + moe_a2a_backend=moe_a2a_backend, + ) + + dense = ("NemotronHForCausalLM", SimpleNamespace()) + rejected = { + "flashinfer_trtllm": (args("flashinfer_trtllm"), *dense), + "cutedsl_without_a2a": (args("flashinfer_cutedsl"), *dense), + "puzzle_mixed_expert_counts": ( + args("flashinfer_trtllm_routed"), + "NemotronHPuzzleForCausalLM", + puzzle(64, 128), + ), + } + accepted = { + "cutedsl_with_a2a": (args("flashinfer_cutedsl", "flashinfer"), *dense), + "puzzle_equal_expert_counts": ( + args("flashinfer_trtllm_routed"), + "NemotronHPuzzleForCausalLM", + puzzle(64, 64), + ), + } + for name, (cfg, model_arch, hf_config) in rejected.items(): + with self.subTest(name), self.assertRaises(ValueError): + _validate_expert_remap(cfg, hf_config, model_arch) + for name, (cfg, model_arch, hf_config) in accepted.items(): + with self.subTest(name): + _validate_expert_remap(cfg, hf_config, model_arch) + def test_nemotron_h_speculation_uses_arch_specific_attention_on_blackwell(self): from sglang.srt.arg_groups.model_overrides.nemotron_h import ( _nemotron_h_overrides, @@ -1525,6 +1600,9 @@ class TestGoldenModelOverrides(_IsolatedPublish): moe_runner_backend="auto", moe_a2a_backend="none", attention_backend=None, + enable_eplb=False, + ep_num_redundant_experts=0, + init_expert_location="trivial", _model_config=model_config, ) @@ -2240,6 +2318,9 @@ class TestGoldenModelOverrides(_IsolatedPublish): speculative_draft_attention_backend=None, page_size=None, mamba_radix_cache_strategy="auto", + enable_eplb=False, + ep_num_redundant_experts=0, + init_expert_location="trivial", _model_config=mc, ) defaults.update(kw)