From 2c739b225b4171fda438e01e69f5f0f4505e7ebd Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:19:33 -0700 Subject: [PATCH] [Fix] Jet-Nemotron build and Granite MoE hybrid final norm (#42306) --- .../kernels/ops/attention/fla/fused_recurrent.py | 6 +++++- python/sglang/srt/models/granitemoehybrid.py | 4 ++-- python/sglang/srt/models/jet_nemotron.py | 16 +++++++++++++--- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/python/sglang/kernels/ops/attention/fla/fused_recurrent.py b/python/sglang/kernels/ops/attention/fla/fused_recurrent.py index 1a59ad0643..9574c7a7b0 100644 --- a/python/sglang/kernels/ops/attention/fla/fused_recurrent.py +++ b/python/sglang/kernels/ops/attention/fla/fused_recurrent.py @@ -1227,7 +1227,11 @@ def fused_recurrent_gated_delta_rule_update( f"The number of initial states is expected to be equal to the number of input sequences, " f"i.e., {len(cu_seqlens) - 1} rather than {initial_state_indices.shape[0]}." ) - if initial_state_indices.shape[0] != intermediate_state_indices.shape[0]: + if ( + intermediate_state_indices is not None + and initial_state_indices.shape[0] + != intermediate_state_indices.shape[0] + ): raise ValueError( f"The number of intermediate state indices is expected to be equal to the number of input sequences, " f"i.e., {initial_state_indices.shape[0]} != {intermediate_state_indices.shape[0]}." diff --git a/python/sglang/srt/models/granitemoehybrid.py b/python/sglang/srt/models/granitemoehybrid.py index e67a4c97f4..95c32c7c96 100644 --- a/python/sglang/srt/models/granitemoehybrid.py +++ b/python/sglang/srt/models/granitemoehybrid.py @@ -386,7 +386,7 @@ class GraniteMoeHybridModel(nn.Module): aux_hidden_states = [] for i in range(self.start_layer, self.end_layer): if i in self.layers_to_capture: - aux_hidden_states.append(hidden_states + residual) + aux_hidden_states.append(hidden_states) layer = self.layers[i] hidden_states, residual = layer( positions, @@ -403,7 +403,7 @@ class GraniteMoeHybridModel(nn.Module): } ) else: - hidden_states, _ = self.norm(hidden_states, residual) + hidden_states = self.norm(hidden_states) if len(aux_hidden_states) == 0: return hidden_states diff --git a/python/sglang/srt/models/jet_nemotron.py b/python/sglang/srt/models/jet_nemotron.py index 5ebf473c2e..e0a1e3f25f 100644 --- a/python/sglang/srt/models/jet_nemotron.py +++ b/python/sglang/srt/models/jet_nemotron.py @@ -32,6 +32,7 @@ from sglang.srt.model_executor.forward_context import get_attn_backend from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.models.qwen2 import Qwen2MLP, Qwen2Model from sglang.srt.utils import add_prefix +from sglang.srt.utils.hf_transformers_utils import get_rope_config class DynamicShortConvolutionKernelGenerator(nn.Module): @@ -286,7 +287,7 @@ class JetBlock(nn.Module): k = nn.functional.silu(k) k = einops.rearrange(k, "l (h d) -> l h d", h=self.num_heads, d=self.head_k_dim) - conv_cache = layer_cache.conv + conv_cache = layer_cache.conv[0] assert isinstance(conv_cache, torch.Tensor) v, new_conv_state = self.dynamic_conv1d( v, @@ -369,12 +370,13 @@ class JetNemotronAttention(nn.Module): prefix=add_prefix("o_proj", prefix), ) + rope_theta, rope_scaling = get_rope_config(self.config) self.rotary_emb = get_rope( self.head_dim, rotary_dim=self.head_dim, max_position=self.config.max_position_embeddings, - base=int(self.config.rope_parameters["rope_theta"]), - rope_scaling=self.config.rope_parameters, + base=int(rope_theta), + rope_scaling=rope_scaling, ) match self.config.layer_types[layer_id]: @@ -420,10 +422,12 @@ class JetNemotronDecoderLayer(nn.Module): config: JetNemotronConfig, alt_stream: torch.cuda.Stream | None = None, layer_id: int = 0, + start_layer: int = 0, quant_config: QuantizationConfig | None = None, prefix: str = "", ) -> None: super().__init__() + self.start_layer = start_layer match config.layer_types[layer_id]: case "attn" | "swa": @@ -521,6 +525,12 @@ class JetNemotronForCausalLM(nn.Module): self.logits_processor = LogitsProcessor(config) self.pooler = Pooler(PoolingType.LAST, normalize=True) + def get_attention_sliding_window_size(self) -> int | None: + swa = self.config.efficient_attention_config.get("swa") + if swa is None or "swa" not in self.config.layer_types: + return None + return swa["window_size"] + @torch.no_grad() def forward( self,