[Perf] dsv4: Keep V4.1 image rows inside the Engram gate kernel (#45904)
This commit is contained in:
@@ -1,5 +1,7 @@
|
||||
"""Fused FP32 Engram gate with a single final cast to the activation dtype."""
|
||||
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
@@ -12,12 +14,15 @@ def _engram_gate_kernel(
|
||||
KV,
|
||||
QW,
|
||||
KW,
|
||||
IDS,
|
||||
O,
|
||||
IMAGE_ID,
|
||||
D: tl.constexpr,
|
||||
HC: tl.constexpr,
|
||||
EPS: tl.constexpr,
|
||||
CLAMP: tl.constexpr,
|
||||
B: tl.constexpr,
|
||||
HAS_IDS: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
token = row // HC
|
||||
@@ -35,7 +40,10 @@ def _engram_gate_kernel(
|
||||
dot = tl.sum((x * weight) * key, 0) * rstd * (D**-0.5)
|
||||
gate = tl.sigmoid(libdevice.copysign(tl.sqrt(tl.maximum(tl.abs(dot), CLAMP)), dot))
|
||||
value = tl.load(KV + token * (HC + 1) * D + HC * D + col, mask, 0).to(tl.float32)
|
||||
tl.store(O + row * D + col, x + gate * value, mask)
|
||||
out = x + gate * value
|
||||
if HAS_IDS:
|
||||
out = tl.where(tl.load(IDS + token) == IMAGE_ID, x, out)
|
||||
tl.store(O + row * D + col, out, mask)
|
||||
|
||||
|
||||
def fused_engram_gate(
|
||||
@@ -45,11 +53,17 @@ def fused_engram_gate(
|
||||
k_weight: torch.Tensor,
|
||||
eps: float,
|
||||
clamp_value: float,
|
||||
input_ids: Optional[torch.Tensor] = None,
|
||||
image_token_id: Optional[int] = None,
|
||||
) -> torch.Tensor:
|
||||
"""Rows whose ``input_ids`` entry is ``image_token_id`` keep ``x`` unchanged."""
|
||||
assert x.ndim == 3 and kv.ndim == 2
|
||||
t, hc, d = x.shape
|
||||
assert kv.shape == (t, (hc + 1) * d)
|
||||
assert q_weight.shape == k_weight.shape == (hc, d)
|
||||
assert input_ids is None or (
|
||||
input_ids.shape == (t,) and input_ids.is_cuda and image_token_id is not None
|
||||
)
|
||||
assert all(
|
||||
a.is_cuda and a.is_contiguous() and a.device == x.device
|
||||
for a in (x, kv, q_weight, k_weight)
|
||||
@@ -64,12 +78,15 @@ def fused_engram_gate(
|
||||
kv,
|
||||
q_weight,
|
||||
k_weight,
|
||||
input_ids,
|
||||
out,
|
||||
image_token_id,
|
||||
d,
|
||||
hc,
|
||||
eps,
|
||||
clamp_value,
|
||||
triton.next_power_of_2(d),
|
||||
input_ids is not None,
|
||||
num_warps=4,
|
||||
enable_fp_fusion=False,
|
||||
)
|
||||
|
||||
@@ -1015,9 +1015,12 @@ def engram_gate(
|
||||
k_weight: torch.Tensor,
|
||||
eps: float,
|
||||
clamp_value: float,
|
||||
input_ids: Optional[torch.Tensor] = None,
|
||||
image_token_id: Optional[int] = None,
|
||||
) -> torch.Tensor:
|
||||
"""x [T, hc_mult, dim]; kv [T, (hc_mult + 1) * dim] holds one key per hc copy
|
||||
followed by the shared value. Adds the gated value to every copy."""
|
||||
followed by the shared value. Adds the gated value to every copy, except in rows
|
||||
whose ``input_ids`` entry is ``image_token_id``, which keep ``x``."""
|
||||
# ``kv`` is the wkv projection of the same tokens, so a shape mismatch is a
|
||||
# caller bug that ``fused_engram_gate`` asserts on, not a reason to fall back.
|
||||
if (
|
||||
@@ -1029,7 +1032,9 @@ def engram_gate(
|
||||
and k_weight.dtype in (torch.bfloat16, torch.float32)
|
||||
and all(t.is_contiguous() for t in (x, kv, q_weight, k_weight))
|
||||
):
|
||||
return fused_engram_gate(x, kv, q_weight, k_weight, eps, clamp_value)
|
||||
return fused_engram_gate(
|
||||
x, kv, q_weight, k_weight, eps, clamp_value, input_ids, image_token_id
|
||||
)
|
||||
hc_mult, dim = x.shape[-2:]
|
||||
key, value = kv.split([hc_mult * dim, dim], dim=-1)
|
||||
key = key.float().unflatten(-1, (hc_mult, dim))
|
||||
@@ -1042,7 +1047,10 @@ def engram_gate(
|
||||
dot = (h * weight * key).sum(-1) * rstd * dim**-0.5
|
||||
# Signed square root before the sigmoid, matching the training kernel.
|
||||
gate = torch.sigmoid(torch.copysign(dot.abs().clamp_min(clamp_value).sqrt(), dot))
|
||||
return (h + gate.unsqueeze(-1) * value.float().unsqueeze(-2)).to(x.dtype)
|
||||
out = (h + gate.unsqueeze(-1) * value.float().unsqueeze(-2)).to(x.dtype)
|
||||
if input_ids is not None:
|
||||
out = torch.where((input_ids == image_token_id)[:, None, None], x, out)
|
||||
return out
|
||||
|
||||
|
||||
class Engram(nn.Module):
|
||||
@@ -1053,9 +1061,12 @@ class Engram(nn.Module):
|
||||
layout: EngramLayout,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
*,
|
||||
image_token_id: Optional[int] = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.layer_hash_index = layout.layer_ids.index(layer_id)
|
||||
self.image_token_id = image_token_id
|
||||
self.eps = config.rms_norm_eps
|
||||
self.clamp_value = 1e-6
|
||||
dim, hc_mult = config.hidden_size, config.hc_mult
|
||||
@@ -1082,9 +1093,11 @@ class Engram(nn.Module):
|
||||
hash_ids: torch.Tensor,
|
||||
forward_batch: Optional[ForwardBatch] = None,
|
||||
*,
|
||||
input_ids: torch.Tensor,
|
||||
cp_all_tokens: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""x [T, hc_mult, dim]; hash_ids [T, n_hash_cols] for this layer."""
|
||||
"""x [T, hc_mult, dim]; hash_ids [T, n_hash_cols] and input_ids [T] for this
|
||||
layer."""
|
||||
# The lookup runs first even for an idle DP-attention batch: under DP
|
||||
# attention it is a collective every rank has to join.
|
||||
emb = self.embed(hash_ids, forward_batch, cp_all_tokens=cp_all_tokens)
|
||||
@@ -1093,6 +1106,14 @@ class Engram(nn.Module):
|
||||
# empty M.
|
||||
return x
|
||||
kv, _ = self.wkv(emb.flatten(-2))
|
||||
image_ids = input_ids if self.image_token_id is not None else None
|
||||
return engram_gate(
|
||||
x, kv, self.q_weight, self.k_weight, self.eps, self.clamp_value
|
||||
x,
|
||||
kv,
|
||||
self.q_weight,
|
||||
self.k_weight,
|
||||
self.eps,
|
||||
self.clamp_value,
|
||||
image_ids,
|
||||
self.image_token_id,
|
||||
)
|
||||
|
||||
@@ -858,6 +858,13 @@ def deepseek_v4_engram_hash_ids(hasher, input_ids: torch.Tensor) -> torch.Tensor
|
||||
bcg_deepseek_v4_engram_hash_ids = eager_on_graph(True)(deepseek_v4_engram_hash_ids)
|
||||
|
||||
|
||||
def _engram_image_token_id(config) -> Optional[int]:
|
||||
"""The image token Engram hashes as blocked and leaves ungated (V4.1 vision)."""
|
||||
if config.model_type == "deepseek_v41" and config.vision_n_layers > 0:
|
||||
return config.image_token_id
|
||||
return None
|
||||
|
||||
|
||||
def deepseek_v4_enter_late_layer_tail(sentinel: torch.Tensor) -> torch.Tensor:
|
||||
# Decoder SWA bounded replay under the prefill CUDA graph: swap the attention
|
||||
# backend onto the tail metadata of the *live* batch (the captured args hold
|
||||
@@ -3095,6 +3102,7 @@ class DeepseekV4DecoderLayerBase(nn.Module):
|
||||
engram_layout,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("engram", prefix),
|
||||
image_token_id=_engram_image_token_id(config),
|
||||
)
|
||||
self._input_layernorm_weight_bf16 = None
|
||||
self._post_attention_layernorm_weight_bf16 = None
|
||||
@@ -3729,12 +3737,7 @@ class DeepseekV4Model(nn.Module):
|
||||
self.engram_hasher = EngramHasher.from_config(
|
||||
config,
|
||||
self.engram_layout,
|
||||
image_token_id=(
|
||||
config.image_token_id
|
||||
if config.model_type == "deepseek_v41"
|
||||
and config.vision_n_layers > 0
|
||||
else None
|
||||
),
|
||||
image_token_id=_engram_image_token_id(config),
|
||||
)
|
||||
|
||||
if not _is_cuda:
|
||||
@@ -4067,17 +4070,9 @@ class DeepseekV4Model(nn.Module):
|
||||
state.residual,
|
||||
hash_ids[:, engram.layer_hash_index],
|
||||
forward_batch,
|
||||
input_ids=input_ids,
|
||||
cp_all_tokens=cp_extend,
|
||||
)
|
||||
if (
|
||||
self.config.model_type == "deepseek_v41"
|
||||
and self.config.vision_n_layers > 0
|
||||
):
|
||||
hidden_states = torch.where(
|
||||
(input_ids == self.config.image_token_id)[:, None, None],
|
||||
state.residual,
|
||||
hidden_states,
|
||||
)
|
||||
state = state.with_residual(hidden_states)
|
||||
if _is_cuda:
|
||||
hidden_states = residual_batch.set_written(state, forward_batch)
|
||||
|
||||
@@ -8,6 +8,7 @@ from unittest.mock import patch
|
||||
import torch
|
||||
|
||||
import sglang.srt.models.deepseek_v4 as dsv4
|
||||
from sglang.srt.layers.engram import engram_gate
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.model_executor.runner.prefill_cuda_graph_runner import (
|
||||
PrefillCudaGraphRunner,
|
||||
@@ -73,12 +74,16 @@ class TestDeepseekV41BCGMultimodal(CustomTestCase):
|
||||
image_id = 42
|
||||
ids = torch.tensor([7, image_id, 9])
|
||||
hidden = torch.arange(6, dtype=torch.float32).reshape(3, 1, 2)
|
||||
kv = torch.ones(3, 4)
|
||||
weight = torch.ones(1, 2)
|
||||
|
||||
class Engram:
|
||||
layer_hash_index = 0
|
||||
|
||||
def __call__(self, residual, *args, **kwargs):
|
||||
return residual + 10
|
||||
def __call__(self, residual, *args, input_ids, **kwargs):
|
||||
return engram_gate(
|
||||
residual, kv, weight, weight, 1e-6, 1e-6, input_ids, image_id
|
||||
)
|
||||
|
||||
layer = SimpleNamespace(
|
||||
engram=Engram(),
|
||||
@@ -97,7 +102,8 @@ class TestDeepseekV41BCGMultimodal(CustomTestCase):
|
||||
model_type="deepseek_v41", vision_n_layers=1, image_token_id=image_id
|
||||
),
|
||||
)
|
||||
expected = hidden + 10
|
||||
expected = engram_gate(hidden, kv, weight, weight, 1e-6, 1e-6)
|
||||
self.assertFalse(torch.equal(expected[1], hidden[1]))
|
||||
expected[1] = hidden[1]
|
||||
for contains_mm in (False, True):
|
||||
batch = SimpleNamespace(
|
||||
|
||||
Reference in New Issue
Block a user