[Fix] Kimi K3 KDA value head size and SiTU input alignment (#42893)

This commit is contained in:
Cheng Wan
2026-10-09 01:25:28 -07:00
committed by GitHub
parent 48a6c0a5d0
commit 666da4f56b
2 changed files with 6 additions and 4 deletions
@@ -58,9 +58,11 @@ def situ_and_mul(
out_dtype = torch.bfloat16 if input.dtype == torch.float32 else input.dtype
out = input.new_empty(*input.shape[:-1], hidden_size, dtype=out_dtype)
# 2D inputs may be row-strided (e.g. a slice of a fused-GEMM output);
# higher-rank inputs keep the dense-view path.
if input.dim() == 2 and input.stride(1) == 1:
# 2D inputs may be row-strided (e.g. a slice of a fused-GEMM output) when
# the row stride is a whole number of the kernel's 32-byte vectors of the
# wider dtype; other inputs keep the dense-view path.
widest = max(input.element_size(), out.element_size())
if input.dim() == 2 and input.stride(1) == 1 and input.stride(0) * widest % 32 == 0:
input_2d = input
else:
input_2d = input.contiguous().view(-1, hidden_size * 2)
+1 -1
View File
@@ -1535,7 +1535,7 @@ class KimiK3DeltaAttention(nn.Module):
self.num_k_heads = config.linear_attn_config["num_heads"]
self.num_v_heads = config.linear_attn_config["num_heads"]
self.head_k_dim = config.linear_attn_config["head_dim"]
self.head_v_dim = config.v_head_dim
self.head_v_dim = config.linear_attn_config["head_dim"]
self.layer_idx = layer_idx
self.prefix = prefix
assert self.num_heads % self.attn_tp_size == 0