[BugFix] Pass token-major Q/K tensors from Gemma-3 to RadixAttention (#37984)

Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
Co-authored-by: Po-Han Huang <pohanh@nvidia.com>
This commit is contained in:
Jan Bernlöhr
2026-09-30 12:05:41 -07:00
committed by GitHub
co-authored by Po-Han Huang Po-Han Huang
parent 3a398442bf
commit 4dfc0ac131
+8 -19
View File
@@ -255,19 +255,16 @@ class Gemma3Attention(nn.Module):
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
# [s, h, head_dim]
q = q.unflatten(-1, (self.num_heads, self.head_dim)).unsqueeze(0)
q = q.unflatten(-1, (self.num_heads, self.head_dim))
q = self.q_norm(q)
k = k.unflatten(-1, (self.num_kv_heads, self.head_dim)).unsqueeze(0)
k = k.unflatten(-1, (self.num_kv_heads, self.head_dim))
k = self.k_norm(k)
q, k = self.rotary_emb(positions, q, k)
attn_output = self.attn(q, k, v, forward_batch=forward_batch)
# Compatible with triton backend which returns [1, s, h, head_dim]
if attn_output.dim() == 4 and attn_output.shape[0] == 1:
attn_output = attn_output.squeeze(0)
attn_output = attn_output.flatten(-2, -1)
# [s, h * head_dim]
attn_output = attn_output.reshape(-1, self.q_size)
output, _ = self.o_proj(attn_output)
return output
@@ -286,29 +283,21 @@ class Gemma3Attention(nn.Module):
# [s, h, head_dim]
q = q.unflatten(-1, (self.num_heads, self.head_dim))
# -> [h, s, head_dim]
q = q.transpose(0, 1).unsqueeze(0)
q = self.q_norm(q)
k = k.unflatten(-1, (self.num_kv_heads, self.head_dim))
# -> [h, s, head_dim]
k = k.transpose(0, 1).unsqueeze(0)
k = self.k_norm(k)
# q, k = self.rotary_emb(positions, q, k)
cos, sin = position_embeddings
# Gemma3RotaryEmbedding preserves the leading position-id dimensions,
# while SGLang represents the corresponding Q/K tensors token-major.
cos = cos.reshape(-1, cos.shape[-1])
sin = sin.reshape(-1, sin.shape[-1])
q, k = apply_rotary_pos_emb(q, k, cos, sin)
# [b, h, s, head_dim] -> [b, s, h, head_dim]
q = q.permute(0, 2, 1, 3)
k = k.permute(0, 2, 1, 3)
attn_output = self.attn(q, k, v, forward_batch=forward_batch)
# Compatible with triton backend which returns [1, s, h, head_dim]
if attn_output.dim() == 4 and attn_output.shape[0] == 1:
attn_output = attn_output.squeeze(0)
attn_output = attn_output.flatten(-2, -1)
# [s, h * head_dim]
attn_output = attn_output.reshape(-1, self.q_size)
output, _ = self.o_proj(attn_output)
return output