[Perf] dsv4: Keep V4.1 image rows inside the Engram gate kernel (#45904)

This commit is contained in:
Liangsheng Yin
2026-10-22 17:05:01 -07:00
committed by GitHub
parent 77f90ade97
commit ae4258715c
4 changed files with 63 additions and 24 deletions
@@ -1,5 +1,7 @@
"""Fused FP32 Engram gate with a single final cast to the activation dtype."""
from typing import Optional
import torch
import triton
import triton.language as tl
@@ -12,12 +14,15 @@ def _engram_gate_kernel(
KV,
QW,
KW,
IDS,
O,
IMAGE_ID,
D: tl.constexpr,
HC: tl.constexpr,
EPS: tl.constexpr,
CLAMP: tl.constexpr,
B: tl.constexpr,
HAS_IDS: tl.constexpr,
):
row = tl.program_id(0)
token = row // HC
@@ -35,7 +40,10 @@ def _engram_gate_kernel(
dot = tl.sum((x * weight) * key, 0) * rstd * (D**-0.5)
gate = tl.sigmoid(libdevice.copysign(tl.sqrt(tl.maximum(tl.abs(dot), CLAMP)), dot))
value = tl.load(KV + token * (HC + 1) * D + HC * D + col, mask, 0).to(tl.float32)
tl.store(O + row * D + col, x + gate * value, mask)
out = x + gate * value
if HAS_IDS:
out = tl.where(tl.load(IDS + token) == IMAGE_ID, x, out)
tl.store(O + row * D + col, out, mask)
def fused_engram_gate(
@@ -45,11 +53,17 @@ def fused_engram_gate(
k_weight: torch.Tensor,
eps: float,
clamp_value: float,
input_ids: Optional[torch.Tensor] = None,
image_token_id: Optional[int] = None,
) -> torch.Tensor:
"""Rows whose ``input_ids`` entry is ``image_token_id`` keep ``x`` unchanged."""
assert x.ndim == 3 and kv.ndim == 2
t, hc, d = x.shape
assert kv.shape == (t, (hc + 1) * d)
assert q_weight.shape == k_weight.shape == (hc, d)
assert input_ids is None or (
input_ids.shape == (t,) and input_ids.is_cuda and image_token_id is not None
)
assert all(
a.is_cuda and a.is_contiguous() and a.device == x.device
for a in (x, kv, q_weight, k_weight)
@@ -64,12 +78,15 @@ def fused_engram_gate(
kv,
q_weight,
k_weight,
input_ids,
out,
image_token_id,
d,
hc,
eps,
clamp_value,
triton.next_power_of_2(d),
input_ids is not None,
num_warps=4,
enable_fp_fusion=False,
)
+26 -5
View File
@@ -1015,9 +1015,12 @@ def engram_gate(
k_weight: torch.Tensor,
eps: float,
clamp_value: float,
input_ids: Optional[torch.Tensor] = None,
image_token_id: Optional[int] = None,
) -> torch.Tensor:
"""x [T, hc_mult, dim]; kv [T, (hc_mult + 1) * dim] holds one key per hc copy
followed by the shared value. Adds the gated value to every copy."""
followed by the shared value. Adds the gated value to every copy, except in rows
whose ``input_ids`` entry is ``image_token_id``, which keep ``x``."""
# ``kv`` is the wkv projection of the same tokens, so a shape mismatch is a
# caller bug that ``fused_engram_gate`` asserts on, not a reason to fall back.
if (
@@ -1029,7 +1032,9 @@ def engram_gate(
and k_weight.dtype in (torch.bfloat16, torch.float32)
and all(t.is_contiguous() for t in (x, kv, q_weight, k_weight))
):
return fused_engram_gate(x, kv, q_weight, k_weight, eps, clamp_value)
return fused_engram_gate(
x, kv, q_weight, k_weight, eps, clamp_value, input_ids, image_token_id
)
hc_mult, dim = x.shape[-2:]
key, value = kv.split([hc_mult * dim, dim], dim=-1)
key = key.float().unflatten(-1, (hc_mult, dim))
@@ -1042,7 +1047,10 @@ def engram_gate(
dot = (h * weight * key).sum(-1) * rstd * dim**-0.5
# Signed square root before the sigmoid, matching the training kernel.
gate = torch.sigmoid(torch.copysign(dot.abs().clamp_min(clamp_value).sqrt(), dot))
return (h + gate.unsqueeze(-1) * value.float().unsqueeze(-2)).to(x.dtype)
out = (h + gate.unsqueeze(-1) * value.float().unsqueeze(-2)).to(x.dtype)
if input_ids is not None:
out = torch.where((input_ids == image_token_id)[:, None, None], x, out)
return out
class Engram(nn.Module):
@@ -1053,9 +1061,12 @@ class Engram(nn.Module):
layout: EngramLayout,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
*,
image_token_id: Optional[int] = None,
):
super().__init__()
self.layer_hash_index = layout.layer_ids.index(layer_id)
self.image_token_id = image_token_id
self.eps = config.rms_norm_eps
self.clamp_value = 1e-6
dim, hc_mult = config.hidden_size, config.hc_mult
@@ -1082,9 +1093,11 @@ class Engram(nn.Module):
hash_ids: torch.Tensor,
forward_batch: Optional[ForwardBatch] = None,
*,
input_ids: torch.Tensor,
cp_all_tokens: bool = False,
) -> torch.Tensor:
"""x [T, hc_mult, dim]; hash_ids [T, n_hash_cols] for this layer."""
"""x [T, hc_mult, dim]; hash_ids [T, n_hash_cols] and input_ids [T] for this
layer."""
# The lookup runs first even for an idle DP-attention batch: under DP
# attention it is a collective every rank has to join.
emb = self.embed(hash_ids, forward_batch, cp_all_tokens=cp_all_tokens)
@@ -1093,6 +1106,14 @@ class Engram(nn.Module):
# empty M.
return x
kv, _ = self.wkv(emb.flatten(-2))
image_ids = input_ids if self.image_token_id is not None else None
return engram_gate(
x, kv, self.q_weight, self.k_weight, self.eps, self.clamp_value
x,
kv,
self.q_weight,
self.k_weight,
self.eps,
self.clamp_value,
image_ids,
self.image_token_id,
)
+10 -15
View File
@@ -858,6 +858,13 @@ def deepseek_v4_engram_hash_ids(hasher, input_ids: torch.Tensor) -> torch.Tensor
bcg_deepseek_v4_engram_hash_ids = eager_on_graph(True)(deepseek_v4_engram_hash_ids)
def _engram_image_token_id(config) -> Optional[int]:
"""The image token Engram hashes as blocked and leaves ungated (V4.1 vision)."""
if config.model_type == "deepseek_v41" and config.vision_n_layers > 0:
return config.image_token_id
return None
def deepseek_v4_enter_late_layer_tail(sentinel: torch.Tensor) -> torch.Tensor:
# Decoder SWA bounded replay under the prefill CUDA graph: swap the attention
# backend onto the tail metadata of the *live* batch (the captured args hold
@@ -3095,6 +3102,7 @@ class DeepseekV4DecoderLayerBase(nn.Module):
engram_layout,
quant_config=quant_config,
prefix=add_prefix("engram", prefix),
image_token_id=_engram_image_token_id(config),
)
self._input_layernorm_weight_bf16 = None
self._post_attention_layernorm_weight_bf16 = None
@@ -3729,12 +3737,7 @@ class DeepseekV4Model(nn.Module):
self.engram_hasher = EngramHasher.from_config(
config,
self.engram_layout,
image_token_id=(
config.image_token_id
if config.model_type == "deepseek_v41"
and config.vision_n_layers > 0
else None
),
image_token_id=_engram_image_token_id(config),
)
if not _is_cuda:
@@ -4067,17 +4070,9 @@ class DeepseekV4Model(nn.Module):
state.residual,
hash_ids[:, engram.layer_hash_index],
forward_batch,
input_ids=input_ids,
cp_all_tokens=cp_extend,
)
if (
self.config.model_type == "deepseek_v41"
and self.config.vision_n_layers > 0
):
hidden_states = torch.where(
(input_ids == self.config.image_token_id)[:, None, None],
state.residual,
hidden_states,
)
state = state.with_residual(hidden_states)
if _is_cuda:
hidden_states = residual_batch.set_written(state, forward_batch)