[BugFix] Pass token-major Q/K tensors from Gemma-3 to RadixAttention (#37984)
Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com> Co-authored-by: Po-Han Huang <pohanh@nvidia.com>
This commit is contained in:
co-authored by
Po-Han Huang
Po-Han Huang
parent
3a398442bf
commit
4dfc0ac131
@@ -255,19 +255,16 @@ class Gemma3Attention(nn.Module):
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
|
||||
# [s, h, head_dim]
|
||||
q = q.unflatten(-1, (self.num_heads, self.head_dim)).unsqueeze(0)
|
||||
q = q.unflatten(-1, (self.num_heads, self.head_dim))
|
||||
q = self.q_norm(q)
|
||||
k = k.unflatten(-1, (self.num_kv_heads, self.head_dim)).unsqueeze(0)
|
||||
k = k.unflatten(-1, (self.num_kv_heads, self.head_dim))
|
||||
k = self.k_norm(k)
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
|
||||
attn_output = self.attn(q, k, v, forward_batch=forward_batch)
|
||||
|
||||
# Compatible with triton backend which returns [1, s, h, head_dim]
|
||||
if attn_output.dim() == 4 and attn_output.shape[0] == 1:
|
||||
attn_output = attn_output.squeeze(0)
|
||||
attn_output = attn_output.flatten(-2, -1)
|
||||
# [s, h * head_dim]
|
||||
attn_output = attn_output.reshape(-1, self.q_size)
|
||||
|
||||
output, _ = self.o_proj(attn_output)
|
||||
return output
|
||||
@@ -286,29 +283,21 @@ class Gemma3Attention(nn.Module):
|
||||
|
||||
# [s, h, head_dim]
|
||||
q = q.unflatten(-1, (self.num_heads, self.head_dim))
|
||||
# -> [h, s, head_dim]
|
||||
q = q.transpose(0, 1).unsqueeze(0)
|
||||
q = self.q_norm(q)
|
||||
k = k.unflatten(-1, (self.num_kv_heads, self.head_dim))
|
||||
# -> [h, s, head_dim]
|
||||
k = k.transpose(0, 1).unsqueeze(0)
|
||||
k = self.k_norm(k)
|
||||
|
||||
# q, k = self.rotary_emb(positions, q, k)
|
||||
cos, sin = position_embeddings
|
||||
# Gemma3RotaryEmbedding preserves the leading position-id dimensions,
|
||||
# while SGLang represents the corresponding Q/K tensors token-major.
|
||||
cos = cos.reshape(-1, cos.shape[-1])
|
||||
sin = sin.reshape(-1, sin.shape[-1])
|
||||
q, k = apply_rotary_pos_emb(q, k, cos, sin)
|
||||
|
||||
# [b, h, s, head_dim] -> [b, s, h, head_dim]
|
||||
q = q.permute(0, 2, 1, 3)
|
||||
k = k.permute(0, 2, 1, 3)
|
||||
|
||||
attn_output = self.attn(q, k, v, forward_batch=forward_batch)
|
||||
|
||||
# Compatible with triton backend which returns [1, s, h, head_dim]
|
||||
if attn_output.dim() == 4 and attn_output.shape[0] == 1:
|
||||
attn_output = attn_output.squeeze(0)
|
||||
attn_output = attn_output.flatten(-2, -1)
|
||||
# [s, h * head_dim]
|
||||
attn_output = attn_output.reshape(-1, self.q_size)
|
||||
|
||||
output, _ = self.o_proj(attn_output)
|
||||
return output
|
||||
|
||||
Reference in New Issue
Block a user