[DSv4.1] Trim prefill model-entry overhead (#41704)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: BBuf <1182563586@qq.com>
This commit is contained in:
DarkSharpness
2026-10-07 09:50:34 +08:00
committed by GitHub
co-authored by Claude Opus 5.5 BBuf
parent 7a4d6dfd2f
commit 2f6e0ec45d
3 changed files with 39 additions and 1 deletions
@@ -1888,6 +1888,34 @@ def hc_expand(x: torch.Tensor, n: int) -> torch.Tensor:
return x.repeat(1, n)
@triton.jit
def _hc_broadcast_kernel(
x_ptr, y_ptr, H, x_stride_m, HC: tl.constexpr, BLOCK_H: tl.constexpr
):
row = tl.program_id(0).to(tl.int64)
offs = tl.program_id(1) * BLOCK_H + tl.arange(0, BLOCK_H)
mask = offs < H
x = tl.load(x_ptr + row * x_stride_m + offs, mask=mask)
y_row = y_ptr + row * (HC * H)
for k in tl.static_range(HC):
tl.store(y_row + k * H + offs, x, mask=mask)
def hc_broadcast(x: torch.Tensor, hc_mult: int) -> torch.Tensor:
"""[T, H] -> [T, hc_mult, H], every copy equal to x: the initial mHC residual."""
if not x.is_cuda or x.stride(-1) != 1:
return x.unsqueeze(1).repeat(1, hc_mult, 1)
num_tokens, hidden = x.shape
y = x.new_empty(num_tokens, hc_mult, hidden)
if num_tokens == 0:
return y
block_h = 1024
_hc_broadcast_kernel[(num_tokens, triton.cdiv(hidden, block_h))](
x, y, hidden, x.stride(0), HC=hc_mult, BLOCK_H=block_h, num_warps=4
)
return y
def hc_contract(x: torch.Tensor, n: int) -> torch.Tensor:
return x.unflatten(-1, (n, -1)).mean(dim=-2)
+5
View File
@@ -247,6 +247,11 @@ class EngramHasher(nn.Module):
self.image_token_id: Optional[int] = None
self.history: Optional[torch.Tensor] = None
self.pad_row = 0
# typing only
self.token_map: torch.Tensor
self.multipliers: torch.Tensor
self.primes: torch.Tensor
self.offsets: torch.Tensor
def init_history(self, num_req_slots: int, device) -> None:
"""Allocate oldest-first history with a spare row for graph padding."""
+6 -1
View File
@@ -4065,9 +4065,12 @@ class DeepseekV4Model(nn.Module):
forward_batch,
cp_all_tokens=cp_extend,
)
# Only multimodal placeholders become image_token_id, so a text-only
# batch skips this full-residual where.
if (
self.config.model_type == "deepseek_v41"
and self.config.vision_n_layers > 0
and forward_batch.contains_mm_inputs()
):
hidden_states = torch.where(
(input_ids == self.config.image_token_id)[:, None, None],
@@ -4244,7 +4247,9 @@ class DeepseekV4Model(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
hidden_states = hidden_states.unsqueeze(1).repeat(1, self.hc_mult, 1)
from sglang.kernels.ops.layernorm.mhc import hc_broadcast
hidden_states = hc_broadcast(hidden_states, self.hc_mult)
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]