[DSv4.1] Trim prefill model-entry overhead (#41704)
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: BBuf <1182563586@qq.com>
(cherry picked from commit 2f6e0ec45d)
This commit is contained in:
@@ -1882,6 +1882,34 @@ def hc_expand(x: torch.Tensor, n: int) -> torch.Tensor:
|
||||
return x.repeat(1, n)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _hc_broadcast_kernel(
|
||||
x_ptr, y_ptr, H, x_stride_m, HC: tl.constexpr, BLOCK_H: tl.constexpr
|
||||
):
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
offs = tl.program_id(1) * BLOCK_H + tl.arange(0, BLOCK_H)
|
||||
mask = offs < H
|
||||
x = tl.load(x_ptr + row * x_stride_m + offs, mask=mask)
|
||||
y_row = y_ptr + row * (HC * H)
|
||||
for k in tl.static_range(HC):
|
||||
tl.store(y_row + k * H + offs, x, mask=mask)
|
||||
|
||||
|
||||
def hc_broadcast(x: torch.Tensor, hc_mult: int) -> torch.Tensor:
|
||||
"""[T, H] -> [T, hc_mult, H], every copy equal to x: the initial mHC residual."""
|
||||
if not x.is_cuda or x.stride(-1) != 1:
|
||||
return x.unsqueeze(1).repeat(1, hc_mult, 1)
|
||||
num_tokens, hidden = x.shape
|
||||
y = x.new_empty(num_tokens, hc_mult, hidden)
|
||||
if num_tokens == 0:
|
||||
return y
|
||||
block_h = 1024
|
||||
_hc_broadcast_kernel[(num_tokens, triton.cdiv(hidden, block_h))](
|
||||
x, y, hidden, x.stride(0), HC=hc_mult, BLOCK_H=block_h, num_warps=4
|
||||
)
|
||||
return y
|
||||
|
||||
|
||||
def hc_contract(x: torch.Tensor, n: int) -> torch.Tensor:
|
||||
return x.unflatten(-1, (n, -1)).mean(dim=-2)
|
||||
|
||||
|
||||
@@ -239,6 +239,11 @@ class EngramHasher(nn.Module):
|
||||
self.image_token_id: Optional[int] = None
|
||||
self.history: Optional[torch.Tensor] = None
|
||||
self.pad_row = 0
|
||||
# typing only
|
||||
self.token_map: torch.Tensor
|
||||
self.multipliers: torch.Tensor
|
||||
self.primes: torch.Tensor
|
||||
self.offsets: torch.Tensor
|
||||
|
||||
def init_history(self, num_req_slots: int, device) -> None:
|
||||
"""Allocate oldest-first history with a spare row for graph padding."""
|
||||
|
||||
@@ -4488,9 +4488,12 @@ class DeepseekV4Model(nn.Module):
|
||||
forward_batch,
|
||||
cp_all_tokens=cp_extend,
|
||||
)
|
||||
# Only multimodal placeholders become image_token_id, so a text-only
|
||||
# batch skips this full-residual where.
|
||||
if (
|
||||
self.config.model_type == "deepseek_v41"
|
||||
and self.config.vision_n_layers > 0
|
||||
and forward_batch.contains_mm_inputs()
|
||||
):
|
||||
hidden_states = torch.where(
|
||||
(input_ids == self.config.image_token_id)[:, None, None],
|
||||
@@ -4715,7 +4718,9 @@ class DeepseekV4Model(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
hidden_states = hidden_states.unsqueeze(1).repeat(1, self.hc_mult, 1)
|
||||
from sglang.kernels.ops.layernorm.mhc import hc_broadcast
|
||||
|
||||
hidden_states = hc_broadcast(hidden_states, self.hc_mult)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
|
||||
Reference in New Issue
Block a user