diff --git a/python/sglang/kernels/ops/embeddings/engram_gate.py b/python/sglang/kernels/ops/embeddings/engram_gate.py index 994c5b7b7a..fab42ed36f 100644 --- a/python/sglang/kernels/ops/embeddings/engram_gate.py +++ b/python/sglang/kernels/ops/embeddings/engram_gate.py @@ -1,5 +1,7 @@ """Fused FP32 Engram gate with a single final cast to the activation dtype.""" +from typing import Optional + import torch import triton import triton.language as tl @@ -12,12 +14,15 @@ def _engram_gate_kernel( KV, QW, KW, + IDS, O, + IMAGE_ID, D: tl.constexpr, HC: tl.constexpr, EPS: tl.constexpr, CLAMP: tl.constexpr, B: tl.constexpr, + HAS_IDS: tl.constexpr, ): row = tl.program_id(0) token = row // HC @@ -35,7 +40,10 @@ def _engram_gate_kernel( dot = tl.sum((x * weight) * key, 0) * rstd * (D**-0.5) gate = tl.sigmoid(libdevice.copysign(tl.sqrt(tl.maximum(tl.abs(dot), CLAMP)), dot)) value = tl.load(KV + token * (HC + 1) * D + HC * D + col, mask, 0).to(tl.float32) - tl.store(O + row * D + col, x + gate * value, mask) + out = x + gate * value + if HAS_IDS: + out = tl.where(tl.load(IDS + token) == IMAGE_ID, x, out) + tl.store(O + row * D + col, out, mask) def fused_engram_gate( @@ -45,11 +53,17 @@ def fused_engram_gate( k_weight: torch.Tensor, eps: float, clamp_value: float, + input_ids: Optional[torch.Tensor] = None, + image_token_id: Optional[int] = None, ) -> torch.Tensor: + """Rows whose ``input_ids`` entry is ``image_token_id`` keep ``x`` unchanged.""" assert x.ndim == 3 and kv.ndim == 2 t, hc, d = x.shape assert kv.shape == (t, (hc + 1) * d) assert q_weight.shape == k_weight.shape == (hc, d) + assert input_ids is None or ( + input_ids.shape == (t,) and input_ids.is_cuda and image_token_id is not None + ) assert all( a.is_cuda and a.is_contiguous() and a.device == x.device for a in (x, kv, q_weight, k_weight) @@ -64,12 +78,15 @@ def fused_engram_gate( kv, q_weight, k_weight, + input_ids, out, + image_token_id, d, hc, eps, clamp_value, triton.next_power_of_2(d), + input_ids is not None, num_warps=4, enable_fp_fusion=False, ) diff --git a/python/sglang/srt/layers/engram.py b/python/sglang/srt/layers/engram.py index 9b3dee7907..c7be800c73 100644 --- a/python/sglang/srt/layers/engram.py +++ b/python/sglang/srt/layers/engram.py @@ -1015,9 +1015,12 @@ def engram_gate( k_weight: torch.Tensor, eps: float, clamp_value: float, + input_ids: Optional[torch.Tensor] = None, + image_token_id: Optional[int] = None, ) -> torch.Tensor: """x [T, hc_mult, dim]; kv [T, (hc_mult + 1) * dim] holds one key per hc copy - followed by the shared value. Adds the gated value to every copy.""" + followed by the shared value. Adds the gated value to every copy, except in rows + whose ``input_ids`` entry is ``image_token_id``, which keep ``x``.""" # ``kv`` is the wkv projection of the same tokens, so a shape mismatch is a # caller bug that ``fused_engram_gate`` asserts on, not a reason to fall back. if ( @@ -1029,7 +1032,9 @@ def engram_gate( and k_weight.dtype in (torch.bfloat16, torch.float32) and all(t.is_contiguous() for t in (x, kv, q_weight, k_weight)) ): - return fused_engram_gate(x, kv, q_weight, k_weight, eps, clamp_value) + return fused_engram_gate( + x, kv, q_weight, k_weight, eps, clamp_value, input_ids, image_token_id + ) hc_mult, dim = x.shape[-2:] key, value = kv.split([hc_mult * dim, dim], dim=-1) key = key.float().unflatten(-1, (hc_mult, dim)) @@ -1042,7 +1047,10 @@ def engram_gate( dot = (h * weight * key).sum(-1) * rstd * dim**-0.5 # Signed square root before the sigmoid, matching the training kernel. gate = torch.sigmoid(torch.copysign(dot.abs().clamp_min(clamp_value).sqrt(), dot)) - return (h + gate.unsqueeze(-1) * value.float().unsqueeze(-2)).to(x.dtype) + out = (h + gate.unsqueeze(-1) * value.float().unsqueeze(-2)).to(x.dtype) + if input_ids is not None: + out = torch.where((input_ids == image_token_id)[:, None, None], x, out) + return out class Engram(nn.Module): @@ -1053,9 +1061,12 @@ class Engram(nn.Module): layout: EngramLayout, quant_config: Optional[QuantizationConfig] = None, prefix: str = "", + *, + image_token_id: Optional[int] = None, ): super().__init__() self.layer_hash_index = layout.layer_ids.index(layer_id) + self.image_token_id = image_token_id self.eps = config.rms_norm_eps self.clamp_value = 1e-6 dim, hc_mult = config.hidden_size, config.hc_mult @@ -1082,9 +1093,11 @@ class Engram(nn.Module): hash_ids: torch.Tensor, forward_batch: Optional[ForwardBatch] = None, *, + input_ids: torch.Tensor, cp_all_tokens: bool = False, ) -> torch.Tensor: - """x [T, hc_mult, dim]; hash_ids [T, n_hash_cols] for this layer.""" + """x [T, hc_mult, dim]; hash_ids [T, n_hash_cols] and input_ids [T] for this + layer.""" # The lookup runs first even for an idle DP-attention batch: under DP # attention it is a collective every rank has to join. emb = self.embed(hash_ids, forward_batch, cp_all_tokens=cp_all_tokens) @@ -1093,6 +1106,14 @@ class Engram(nn.Module): # empty M. return x kv, _ = self.wkv(emb.flatten(-2)) + image_ids = input_ids if self.image_token_id is not None else None return engram_gate( - x, kv, self.q_weight, self.k_weight, self.eps, self.clamp_value + x, + kv, + self.q_weight, + self.k_weight, + self.eps, + self.clamp_value, + image_ids, + self.image_token_id, ) diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index 4379714115..f8e52b948d 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -858,6 +858,13 @@ def deepseek_v4_engram_hash_ids(hasher, input_ids: torch.Tensor) -> torch.Tensor bcg_deepseek_v4_engram_hash_ids = eager_on_graph(True)(deepseek_v4_engram_hash_ids) +def _engram_image_token_id(config) -> Optional[int]: + """The image token Engram hashes as blocked and leaves ungated (V4.1 vision).""" + if config.model_type == "deepseek_v41" and config.vision_n_layers > 0: + return config.image_token_id + return None + + def deepseek_v4_enter_late_layer_tail(sentinel: torch.Tensor) -> torch.Tensor: # Decoder SWA bounded replay under the prefill CUDA graph: swap the attention # backend onto the tail metadata of the *live* batch (the captured args hold @@ -3095,6 +3102,7 @@ class DeepseekV4DecoderLayerBase(nn.Module): engram_layout, quant_config=quant_config, prefix=add_prefix("engram", prefix), + image_token_id=_engram_image_token_id(config), ) self._input_layernorm_weight_bf16 = None self._post_attention_layernorm_weight_bf16 = None @@ -3729,12 +3737,7 @@ class DeepseekV4Model(nn.Module): self.engram_hasher = EngramHasher.from_config( config, self.engram_layout, - image_token_id=( - config.image_token_id - if config.model_type == "deepseek_v41" - and config.vision_n_layers > 0 - else None - ), + image_token_id=_engram_image_token_id(config), ) if not _is_cuda: @@ -4067,17 +4070,9 @@ class DeepseekV4Model(nn.Module): state.residual, hash_ids[:, engram.layer_hash_index], forward_batch, + input_ids=input_ids, cp_all_tokens=cp_extend, ) - if ( - self.config.model_type == "deepseek_v41" - and self.config.vision_n_layers > 0 - ): - hidden_states = torch.where( - (input_ids == self.config.image_token_id)[:, None, None], - state.residual, - hidden_states, - ) state = state.with_residual(hidden_states) if _is_cuda: hidden_states = residual_batch.set_written(state, forward_batch) diff --git a/test/registered/unit/models/test_deepseek_v41_bcg_multimodal.py b/test/registered/unit/models/test_deepseek_v41_bcg_multimodal.py index dab7f2ef83..5446392690 100644 --- a/test/registered/unit/models/test_deepseek_v41_bcg_multimodal.py +++ b/test/registered/unit/models/test_deepseek_v41_bcg_multimodal.py @@ -8,6 +8,7 @@ from unittest.mock import patch import torch import sglang.srt.models.deepseek_v4 as dsv4 +from sglang.srt.layers.engram import engram_gate from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.model_executor.runner.prefill_cuda_graph_runner import ( PrefillCudaGraphRunner, @@ -73,12 +74,16 @@ class TestDeepseekV41BCGMultimodal(CustomTestCase): image_id = 42 ids = torch.tensor([7, image_id, 9]) hidden = torch.arange(6, dtype=torch.float32).reshape(3, 1, 2) + kv = torch.ones(3, 4) + weight = torch.ones(1, 2) class Engram: layer_hash_index = 0 - def __call__(self, residual, *args, **kwargs): - return residual + 10 + def __call__(self, residual, *args, input_ids, **kwargs): + return engram_gate( + residual, kv, weight, weight, 1e-6, 1e-6, input_ids, image_id + ) layer = SimpleNamespace( engram=Engram(), @@ -97,7 +102,8 @@ class TestDeepseekV41BCGMultimodal(CustomTestCase): model_type="deepseek_v41", vision_n_layers=1, image_token_id=image_id ), ) - expected = hidden + 10 + expected = engram_gate(hidden, kv, weight, weight, 1e-6, 1e-6) + self.assertFalse(torch.equal(expected[1], hidden[1])) expected[1] = hidden[1] for contains_mm in (False, True): batch = SimpleNamespace(