Support EPLB for Nemotron-H (#43685)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
elvischenv
2026-10-15 13:50:09 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent c5bf3aafbc
commit 6fe5596ebb
7 changed files with 312 additions and 20 deletions
@@ -54,6 +54,14 @@ def _nemotron_h_overrides(server_args: Any, hf_config: Any) -> dict:
quantization = model_config.quantization
overrides["quantization"] = quantization
uses_expert_remap = (
cfg.enable_eplb
or cfg.ep_num_redundant_experts > 0
or cfg.init_expert_location != "trivial"
)
if uses_expert_remap:
_validate_expert_remap(cfg, hf_config, model_arch)
has_w4a16_moe_layers = False
if is_modelopt and quantization == "modelopt_mixed":
has_w4a16_moe_layers = any(
@@ -76,6 +84,18 @@ def _nemotron_h_overrides(server_args: Any, hf_config: Any) -> dict:
"Use marlin as MoE runner backend for "
f"{model_arch} with W4A16_NVFP4 MoE layers"
)
elif (
uses_expert_remap
and cfg.moe_runner_backend == "auto"
and get_platform().is_sm100
and cfg.moe_a2a_backend == "none"
):
# flashinfer_trtllm routes inside the kernel and cannot take the remap.
overrides["moe_runner_backend"] = "flashinfer_trtllm_routed"
logger.info(
"Use flashinfer_trtllm_routed as MoE runner backend on sm100 for "
f"{model_arch} with EPLB or redundant experts"
)
elif (is_modelopt or model_config.quantization is None) and (
cfg.moe_runner_backend == "auto"
):
@@ -127,3 +147,34 @@ def _nemotron_h_overrides(server_args: Any, hf_config: Any) -> dict:
elif get_platform().is_sm100:
overrides["attention_backend"] = "trtllm_mha"
return overrides
def _validate_expert_remap(cfg: Any, hf_config: Any, model_arch: str) -> None:
if cfg.moe_runner_backend == "flashinfer_trtllm":
raise ValueError(
"flashinfer_trtllm routes inside the kernel and does not support EPLB or "
"redundant experts on Nemotron-H; use "
"--moe-runner-backend flashinfer_trtllm_routed."
)
if (
cfg.enable_eplb
and cfg.moe_runner_backend == "flashinfer_cutedsl"
and cfg.moe_a2a_backend == "none"
):
raise ValueError(
"flashinfer_cutedsl without an a2a backend does not migrate its expert "
"scales on EPLB rebalance; use --moe-runner-backend "
"flashinfer_trtllm_routed or set --moe-a2a-backend."
)
if model_arch == "NemotronHPuzzleForCausalLM":
num_routed_experts = {
block["n_routed_experts"]
for block in hf_config.block_configs
if block["block_type"] == "moe"
}
if len(num_routed_experts) > 1:
raise ValueError(
"EPLB and redundant experts need the same routed expert count on "
"every MoE layer, got "
f"{sorted(num_routed_experts)}."
)
@@ -128,6 +128,16 @@ class NemotronH_Nano_VL_V2(EVS):
self.config = config
@property
def routed_experts_weights_of_layer(self):
return self.language_model.routed_experts_weights_of_layer
@classmethod
def get_model_config_for_expert_location(cls, config):
return NemotronHForCausalLM.get_model_config_for_expert_location(
config.llm_config
)
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
im_start_id: int = mm_inputs.im_start_id
im_end_id: int = mm_inputs.im_end_id
+87 -9
View File
@@ -26,6 +26,9 @@ from torch import nn
from sglang.srt.compilation.compilation_config import register_split_op
from sglang.srt.configs import NemotronHConfig
from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MLP, MOE
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import ReLU2
from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
HybridLinearAttnBackend,
@@ -50,6 +53,7 @@ from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.moe.topk import TopK
from sglang.srt.layers.moe.utils import (
RoutingMethodType,
filter_moe_weight_param_global_expert,
get_moe_a2a_backend,
)
from sglang.srt.layers.quantization import QuantizationConfig
@@ -82,6 +86,7 @@ from sglang.srt.models.nemotron_h_utils import make_stage_boundary, stage_facts
from sglang.srt.models.utils import WeightsMapper
from sglang.srt.runtime_context import get_exec, get_parallel
from sglang.srt.utils import (
LazyValue,
add_prefix,
get_current_device_stream_fast,
is_cuda,
@@ -170,9 +175,12 @@ class NemotronHMoE(nn.Module):
layer_idx: int,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
is_nextn: bool = False,
) -> None:
super().__init__()
self.layer_id = layer_idx
self.is_nextn = is_nextn
self.routed_scaling_factor = config.routed_scaling_factor
self.device_module = torch.get_device_module()
@@ -279,16 +287,20 @@ class NemotronHMoE(nn.Module):
def _forward_core(
self,
hidden_states: torch.Tensor,
num_token_non_padded: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
if _is_cuda and (
not get_moe_a2a_backend().is_flashinfer() or get_is_capture_mode()
):
return self._forward_core_shared_routed_overlap(hidden_states)
return self._forward_core_normal(hidden_states)
return self._forward_core_shared_routed_overlap(
hidden_states, num_token_non_padded
)
return self._forward_core_normal(hidden_states, num_token_non_padded)
def _forward_core_normal(
self,
hidden_states: torch.Tensor,
num_token_non_padded: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
# router_scores: [num_tokens, num_experts]
# bf16 gemm on tensor cores with fp32 accumulation/output for sigmoid/topk.
@@ -299,7 +311,16 @@ class NemotronHMoE(nn.Module):
shared_output = self.shared_experts(hidden_states)
else:
shared_output = None
topk_output = self.topk(hidden_states, router_logits)
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=(
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
if not self.is_nextn
else None
),
)
if self.use_latent_moe:
hidden_states = self._apply_fc1_latent_proj(hidden_states)
final_hidden_states = self.experts(hidden_states, topk_output)
@@ -308,6 +329,7 @@ class NemotronHMoE(nn.Module):
def _forward_core_shared_routed_overlap(
self,
hidden_states: torch.Tensor,
num_token_non_padded: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
alt_stream = _get_or_create_alt_stream(self.device_module)
@@ -324,7 +346,16 @@ class NemotronHMoE(nn.Module):
router_logits = torch.mm(
hidden_states, self.gate.weight.t(), out_dtype=torch.float32
)
topk_output = self.topk(hidden_states, router_logits)
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=(
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
if not self.is_nextn
else None
),
)
if self.use_latent_moe:
hidden_states = self._apply_fc1_latent_proj(hidden_states)
final_hidden_states = self.experts(hidden_states, topk_output)
@@ -332,6 +363,16 @@ class NemotronHMoE(nn.Module):
return final_hidden_states, shared_output
def get_moe_weights(self) -> list[torch.Tensor]:
return [
x.data
for name, x in self.experts.named_parameters()
if name not in ["correction_bias"]
and filter_moe_weight_param_global_expert(
name, x, self.experts.num_local_experts
)
]
def _apply_latent_projection(
self,
final_hidden_states: torch.Tensor,
@@ -351,11 +392,19 @@ class NemotronHMoE(nn.Module):
def forward(
self,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch | None = None,
) -> torch.Tensor:
num_tokens, hidden_dim = hidden_states.shape
# routed_scaling_factor is fused into the experts call (applied by the
# MoE runner / topk), so final_hidden_states is already scaled.
final_hidden_states, shared_output = self._forward_core(hidden_states)
final_hidden_states, shared_output = self._forward_core(
hidden_states,
num_token_non_padded=(
forward_batch.moe_num_token_non_padded()
if forward_batch is not None
else None
),
)
if self.use_latent_moe:
final_hidden_states = self._apply_latent_projection(
@@ -430,6 +479,7 @@ class NemotronHMoEDecoderLayer(NemotronHMLPLikeDecoderLayer):
layer_idx: int,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
is_nextn: bool = False,
) -> None:
super().__init__()
layer_config = config.get_nemotron_h_config_for_layer(layer_idx)
@@ -440,11 +490,22 @@ class NemotronHMoEDecoderLayer(NemotronHMLPLikeDecoderLayer):
layer_idx=layer_idx,
quant_config=quant_config,
prefix=f"{prefix}.mixer",
is_nextn=is_nextn,
)
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
self._init_stage_boundary(config, layer_idx)
def forward(
self,
*,
hidden_states,
forward_batch: ForwardBatch,
):
hidden_states = self.boundary.prepare(hidden_states, forward_batch)
hidden_states = self.mixer.forward(hidden_states, forward_batch=forward_batch)
return self.boundary.finish(hidden_states, forward_batch)
class NemotronHAttnLikeDecoderLayer(nn.Module):
"""A mixer (Mamba / attention) stage. Before an FFN stage it leaves its
@@ -729,10 +790,11 @@ class NemotronHModel(nn.Module):
layer = self.layers[i]
if not isinstance(layer, Layers):
raise ValueError(f"Unknown layer type: {type(layer)}")
hidden_states = layer.forward(
hidden_states=hidden_states,
forward_batch=forward_batch,
)
with get_global_expert_distribution_recorder().with_current_layer(i):
hidden_states = layer.forward(
hidden_states=hidden_states,
forward_batch=forward_batch,
)
if not self.pp_group.is_last_rank:
return residual_batch.to_pp(hidden_states, forward_batch)
@@ -842,6 +904,22 @@ class NemotronHForCausalLM(nn.Module):
self.logits_processor = LogitsProcessor(config)
self.capture_aux_hidden_states = False
self.routed_experts_weights_of_layer = LazyValue(
lambda: {
layer_id: self.model.layers[layer_id].mixer.get_moe_weights()
for layer_id in range(self.model.start_layer, self.model.end_layer)
if isinstance(self.model.layers[layer_id], NemotronHMoEDecoderLayer)
}
)
@classmethod
def get_model_config_for_expert_location(cls, config):
return ModelConfigForExpertLocation(
num_layers=config.num_hidden_layers,
num_logical_experts=config.max_n_routed_experts,
num_groups=config.n_group,
)
def _init_model(
self,
config: NemotronHConfig,
+9 -6
View File
@@ -18,6 +18,7 @@ import torch
from torch import nn
from sglang.srt.configs import NemotronHConfig
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary import layer_stack
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
@@ -137,6 +138,7 @@ class NemotronHMTPMoEDecoderLayer(NemotronHMoEDecoderLayer):
layer_idx=layer_idx,
quant_config=quant_config,
prefix=prefix,
is_nextn=True,
)
self.has_start_projections = has_start_projections
self.has_end_norm = has_end_norm
@@ -302,12 +304,13 @@ class NemotronHMultiTokenPredictor(nn.Module):
hidden_states = forward_batch.spec_info.hidden_states
residual_batch.start(forward_batch)
for i in range(self.pattern_len):
hidden_states = self.layers[str(i)](
inputs_embeds=inputs_embeds,
hidden_states=hidden_states,
forward_batch=forward_batch,
)
with get_global_expert_distribution_recorder().disable_this_region():
for i in range(self.pattern_len):
hidden_states = self.layers[str(i)](
inputs_embeds=inputs_embeds,
hidden_states=hidden_states,
forward_batch=forward_batch,
)
return residual_batch.take_output(hidden_states, forward_batch)