From 1b70fc192c7499efd191bcfe10270d406e3fb1a3 Mon Sep 17 00:00:00 2001 From: DarkSharpness <76582120+DarkSharpness@users.noreply.github.com> Date: Wed, 7 Oct 2026 09:50:34 +0800 Subject: [PATCH] [DSv4.1] Trim prefill model-entry overhead (#41704) Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: BBuf <1182563586@qq.com> (cherry picked from commit 2f6e0ec45dd8f3bbcd28e3ec97ea47aa9b1fc3e0) --- python/sglang/kernels/ops/layernorm/mhc.py | 28 ++++++++++++++++++++++ python/sglang/srt/layers/engram.py | 5 ++++ python/sglang/srt/models/deepseek_v4.py | 7 +++++- 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/python/sglang/kernels/ops/layernorm/mhc.py b/python/sglang/kernels/ops/layernorm/mhc.py index e346e38773..dae2556e70 100644 --- a/python/sglang/kernels/ops/layernorm/mhc.py +++ b/python/sglang/kernels/ops/layernorm/mhc.py @@ -1888,6 +1888,34 @@ def hc_expand(x: torch.Tensor, n: int) -> torch.Tensor: return x.repeat(1, n) +@triton.jit +def _hc_broadcast_kernel( + x_ptr, y_ptr, H, x_stride_m, HC: tl.constexpr, BLOCK_H: tl.constexpr +): + row = tl.program_id(0).to(tl.int64) + offs = tl.program_id(1) * BLOCK_H + tl.arange(0, BLOCK_H) + mask = offs < H + x = tl.load(x_ptr + row * x_stride_m + offs, mask=mask) + y_row = y_ptr + row * (HC * H) + for k in tl.static_range(HC): + tl.store(y_row + k * H + offs, x, mask=mask) + + +def hc_broadcast(x: torch.Tensor, hc_mult: int) -> torch.Tensor: + """[T, H] -> [T, hc_mult, H], every copy equal to x: the initial mHC residual.""" + if not x.is_cuda or x.stride(-1) != 1: + return x.unsqueeze(1).repeat(1, hc_mult, 1) + num_tokens, hidden = x.shape + y = x.new_empty(num_tokens, hc_mult, hidden) + if num_tokens == 0: + return y + block_h = 1024 + _hc_broadcast_kernel[(num_tokens, triton.cdiv(hidden, block_h))]( + x, y, hidden, x.stride(0), HC=hc_mult, BLOCK_H=block_h, num_warps=4 + ) + return y + + def hc_contract(x: torch.Tensor, n: int) -> torch.Tensor: return x.unflatten(-1, (n, -1)).mean(dim=-2) diff --git a/python/sglang/srt/layers/engram.py b/python/sglang/srt/layers/engram.py index 0cb2e953c1..4057c669d8 100644 --- a/python/sglang/srt/layers/engram.py +++ b/python/sglang/srt/layers/engram.py @@ -239,6 +239,11 @@ class EngramHasher(nn.Module): self.image_token_id: Optional[int] = None self.history: Optional[torch.Tensor] = None self.pad_row = 0 + # typing only + self.token_map: torch.Tensor + self.multipliers: torch.Tensor + self.primes: torch.Tensor + self.offsets: torch.Tensor def init_history(self, num_req_slots: int, device) -> None: """Allocate oldest-first history with a spare row for graph padding.""" diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index 63345e7543..b329f5f281 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -4497,9 +4497,12 @@ class DeepseekV4Model(nn.Module): forward_batch, cp_all_tokens=cp_extend, ) + # Only multimodal placeholders become image_token_id, so a text-only + # batch skips this full-residual where. if ( self.config.model_type == "deepseek_v41" and self.config.vision_n_layers > 0 + and forward_batch.contains_mm_inputs() ): hidden_states = torch.where( (input_ids == self.config.image_token_id)[:, None, None], @@ -4724,7 +4727,9 @@ class DeepseekV4Model(nn.Module): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - hidden_states = hidden_states.unsqueeze(1).repeat(1, self.hc_mult, 1) + from sglang.kernels.ops.layernorm.mhc import hc_broadcast + + hidden_states = hc_broadcast(hidden_states, self.hc_mult) else: assert pp_proxy_tensors is not None hidden_states = pp_proxy_tensors["hidden_states"]