[Fix] Reduce Nemotron MTP attention outputs once (#40800)

This commit is contained in:
Cheng Wan
2026-09-23 14:39:04 -07:00
committed by GitHub
parent 0e80c73c92
commit 9ebe42238b
2 changed files with 94 additions and 16 deletions
+1 -16
View File
@@ -18,10 +18,7 @@ import torch
from torch import nn
from sglang.srt.configs import NemotronHConfig
from sglang.srt.layers.dp_attention import (
attn_tp_all_reduce,
is_dp_attention_enabled,
)
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import ColumnParallelLinear
from sglang.srt.layers.logits_processor import LogitsProcessor
@@ -37,7 +34,6 @@ from sglang.srt.models.nemotron_h import (
NemotronHForCausalLM,
NemotronHMoEDecoderLayer,
)
from sglang.srt.models.nemotron_h_utils import is_attn_layer
from sglang.srt.models.utils import WeightsMapper
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import add_prefix
@@ -142,10 +138,6 @@ class NemotronHMTPMoEDecoderLayer(NemotronHMoEDecoderLayer):
)
self.has_start_projections = has_start_projections
self.has_end_norm = has_end_norm
_pat = config.mtp_hybrid_override_pattern
self.prev_layer_is_attn = layer_idx > 0 and is_attn_layer(
_pat[(layer_idx - 1) % len(_pat)]
)
self.layer_communicator.is_last_layer = True
if has_start_projections:
@@ -194,13 +186,6 @@ class NemotronHMTPMoEDecoderLayer(NemotronHMoEDecoderLayer):
hidden_states, dim=-1
)
if (
is_dp_attention_enabled()
and self.prev_layer_is_attn
and residual is not None
):
hidden_states = attn_tp_all_reduce(hidden_states)
hidden_states, residual = super().forward(
hidden_states=hidden_states,
residual=residual,