[Fix] Jet-Nemotron build and Granite MoE hybrid final norm (#42306)
This commit is contained in:
@@ -1227,7 +1227,11 @@ def fused_recurrent_gated_delta_rule_update(
|
||||
f"The number of initial states is expected to be equal to the number of input sequences, "
|
||||
f"i.e., {len(cu_seqlens) - 1} rather than {initial_state_indices.shape[0]}."
|
||||
)
|
||||
if initial_state_indices.shape[0] != intermediate_state_indices.shape[0]:
|
||||
if (
|
||||
intermediate_state_indices is not None
|
||||
and initial_state_indices.shape[0]
|
||||
!= intermediate_state_indices.shape[0]
|
||||
):
|
||||
raise ValueError(
|
||||
f"The number of intermediate state indices is expected to be equal to the number of input sequences, "
|
||||
f"i.e., {initial_state_indices.shape[0]} != {intermediate_state_indices.shape[0]}."
|
||||
|
||||
@@ -386,7 +386,7 @@ class GraniteMoeHybridModel(nn.Module):
|
||||
aux_hidden_states = []
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
if i in self.layers_to_capture:
|
||||
aux_hidden_states.append(hidden_states + residual)
|
||||
aux_hidden_states.append(hidden_states)
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(
|
||||
positions,
|
||||
@@ -403,7 +403,7 @@ class GraniteMoeHybridModel(nn.Module):
|
||||
}
|
||||
)
|
||||
else:
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
hidden_states = self.norm(hidden_states)
|
||||
|
||||
if len(aux_hidden_states) == 0:
|
||||
return hidden_states
|
||||
|
||||
@@ -32,6 +32,7 @@ from sglang.srt.model_executor.forward_context import get_attn_backend
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.models.qwen2 import Qwen2MLP, Qwen2Model
|
||||
from sglang.srt.utils import add_prefix
|
||||
from sglang.srt.utils.hf_transformers_utils import get_rope_config
|
||||
|
||||
|
||||
class DynamicShortConvolutionKernelGenerator(nn.Module):
|
||||
@@ -286,7 +287,7 @@ class JetBlock(nn.Module):
|
||||
k = nn.functional.silu(k)
|
||||
k = einops.rearrange(k, "l (h d) -> l h d", h=self.num_heads, d=self.head_k_dim)
|
||||
|
||||
conv_cache = layer_cache.conv
|
||||
conv_cache = layer_cache.conv[0]
|
||||
assert isinstance(conv_cache, torch.Tensor)
|
||||
v, new_conv_state = self.dynamic_conv1d(
|
||||
v,
|
||||
@@ -369,12 +370,13 @@ class JetNemotronAttention(nn.Module):
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
)
|
||||
|
||||
rope_theta, rope_scaling = get_rope_config(self.config)
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
rotary_dim=self.head_dim,
|
||||
max_position=self.config.max_position_embeddings,
|
||||
base=int(self.config.rope_parameters["rope_theta"]),
|
||||
rope_scaling=self.config.rope_parameters,
|
||||
base=int(rope_theta),
|
||||
rope_scaling=rope_scaling,
|
||||
)
|
||||
|
||||
match self.config.layer_types[layer_id]:
|
||||
@@ -420,10 +422,12 @@ class JetNemotronDecoderLayer(nn.Module):
|
||||
config: JetNemotronConfig,
|
||||
alt_stream: torch.cuda.Stream | None = None,
|
||||
layer_id: int = 0,
|
||||
start_layer: int = 0,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.start_layer = start_layer
|
||||
|
||||
match config.layer_types[layer_id]:
|
||||
case "attn" | "swa":
|
||||
@@ -521,6 +525,12 @@ class JetNemotronForCausalLM(nn.Module):
|
||||
self.logits_processor = LogitsProcessor(config)
|
||||
self.pooler = Pooler(PoolingType.LAST, normalize=True)
|
||||
|
||||
def get_attention_sliding_window_size(self) -> int | None:
|
||||
swa = self.config.efficient_attention_config.get("swa")
|
||||
if swa is None or "swa" not in self.config.layer_types:
|
||||
return None
|
||||
return swa["window_size"]
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user