[Fix] Jet-Nemotron build and Granite MoE hybrid final norm (#42306)

This commit is contained in:
Cheng Wan
2026-10-03 02:19:33 -07:00
committed by GitHub
parent b7fc516986
commit 2c739b225b
3 changed files with 20 additions and 6 deletions
@@ -1227,7 +1227,11 @@ def fused_recurrent_gated_delta_rule_update(
f"The number of initial states is expected to be equal to the number of input sequences, "
f"i.e., {len(cu_seqlens) - 1} rather than {initial_state_indices.shape[0]}."
)
if initial_state_indices.shape[0] != intermediate_state_indices.shape[0]:
if (
intermediate_state_indices is not None
and initial_state_indices.shape[0]
!= intermediate_state_indices.shape[0]
):
raise ValueError(
f"The number of intermediate state indices is expected to be equal to the number of input sequences, "
f"i.e., {initial_state_indices.shape[0]} != {intermediate_state_indices.shape[0]}."
+2 -2
View File
@@ -386,7 +386,7 @@ class GraniteMoeHybridModel(nn.Module):
aux_hidden_states = []
for i in range(self.start_layer, self.end_layer):
if i in self.layers_to_capture:
aux_hidden_states.append(hidden_states + residual)
aux_hidden_states.append(hidden_states)
layer = self.layers[i]
hidden_states, residual = layer(
positions,
@@ -403,7 +403,7 @@ class GraniteMoeHybridModel(nn.Module):
}
)
else:
hidden_states, _ = self.norm(hidden_states, residual)
hidden_states = self.norm(hidden_states)
if len(aux_hidden_states) == 0:
return hidden_states
+13 -3
View File
@@ -32,6 +32,7 @@ from sglang.srt.model_executor.forward_context import get_attn_backend
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.models.qwen2 import Qwen2MLP, Qwen2Model
from sglang.srt.utils import add_prefix
from sglang.srt.utils.hf_transformers_utils import get_rope_config
class DynamicShortConvolutionKernelGenerator(nn.Module):
@@ -286,7 +287,7 @@ class JetBlock(nn.Module):
k = nn.functional.silu(k)
k = einops.rearrange(k, "l (h d) -> l h d", h=self.num_heads, d=self.head_k_dim)
conv_cache = layer_cache.conv
conv_cache = layer_cache.conv[0]
assert isinstance(conv_cache, torch.Tensor)
v, new_conv_state = self.dynamic_conv1d(
v,
@@ -369,12 +370,13 @@ class JetNemotronAttention(nn.Module):
prefix=add_prefix("o_proj", prefix),
)
rope_theta, rope_scaling = get_rope_config(self.config)
self.rotary_emb = get_rope(
self.head_dim,
rotary_dim=self.head_dim,
max_position=self.config.max_position_embeddings,
base=int(self.config.rope_parameters["rope_theta"]),
rope_scaling=self.config.rope_parameters,
base=int(rope_theta),
rope_scaling=rope_scaling,
)
match self.config.layer_types[layer_id]:
@@ -420,10 +422,12 @@ class JetNemotronDecoderLayer(nn.Module):
config: JetNemotronConfig,
alt_stream: torch.cuda.Stream | None = None,
layer_id: int = 0,
start_layer: int = 0,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
self.start_layer = start_layer
match config.layer_types[layer_id]:
case "attn" | "swa":
@@ -521,6 +525,12 @@ class JetNemotronForCausalLM(nn.Module):
self.logits_processor = LogitsProcessor(config)
self.pooler = Pooler(PoolingType.LAST, normalize=True)
def get_attention_sliding_window_size(self) -> int | None:
swa = self.config.efficient_attention_config.get("swa")
if swa is None or "swa" not in self.config.layer_types:
return None
return swa["window_size"]
@torch.no_grad()
def forward(
self,