[Perf] deepseek-v4.1: Drop host syncs from encoder-replay folding and tail metadata (#45275)

This commit is contained in:
Liangsheng Yin
2026-10-19 19:39:39 -07:00
committed by GitHub
parent af856717c2
commit 5e630fa571
4 changed files with 29 additions and 11 deletions
@@ -7,6 +7,8 @@ import torch
import triton
import triton.language as tl
from sglang.srt.utils.common import async_h2d
def _inputs_on_cuda(*args, **kwargs) -> bool:
"""Route kernel dispatch by input placement: the first tensor argument
@@ -480,7 +482,9 @@ def late_layer_tail_layout(
floor = torch.full((t,), max(0, s - tail_len), dtype=torch.int32, device=device)
return torch.arange(n - t, n, device=device), tail_lens_cpu, floor
# One H2D copy for the three length vectors; launch count does not grow with bs.
lens = torch.tensor([extend_lens_cpu, tail_lens_cpu, seq_lens_cpu], device=device)
lens = async_h2d(
[extend_lens_cpu, tail_lens_cpu, seq_lens_cpu], dtype=torch.int64, device=device
)
extend_lens, tail_lens, seq_lens = lens[0], lens[1], lens[2]
total = sum(tail_lens_cpu)
req = torch.repeat_interleave(
@@ -1836,7 +1836,7 @@ class DeepseekV4AttnBackend(
token_indices=token_indices,
contiguous_start=contiguous_start,
)
tail_lens = torch.tensor(tail_lens_cpu, dtype=torch.int32, device=device)
tail_lens = async_h2d(tail_lens_cpu, dtype=torch.int32, device=device)
cp_tail = (
self._late_layer_tail_cp_layout(forward_batch, token_indices, tail_lens)
if is_cp_active(forward_batch)
@@ -1876,6 +1876,7 @@ class DeepseekV4AttnBackend(
metadata.low_ratio_req_indices = torch.repeat_interleave(
forward_batch.req_pool_indices.to(torch.int64),
tail_lens.to(torch.int64),
output_size=sum(tail_lens_cpu),
)
positions = _tail_rows(
forward_batch.positions,
@@ -4,6 +4,8 @@ from typing import Any, Optional
import msgspec
import torch
from sglang.srt.utils.common import async_h2d, is_pin_memory_available
class FoldedExtend(msgspec.Struct, frozen=True):
"""An extend batch whose prefix hits start at their replay start."""
@@ -65,7 +67,8 @@ def drop_folded_rows(
# Decoder-SWA tail rows index the folded extend; keep those on original rows.
keep = _tail_rows_on_original(
folded=folded, tail_lens=tail_lens, num_tail_rows=token_indices.shape[0]
).to(token_indices.device, non_blocking=True)
)
keep = _to_device(keep, token_indices.device)
logits_output.hidden_states_token_indices = folded.scheduled_row[
token_indices[keep]
]
@@ -187,7 +190,7 @@ def _engram_history(*, reqs, starts, runner):
for r, s in zip(reqs, starts):
ids = r.full_untruncated_fill_ids[max(0, s - n) : s]
hist.append(hasher.history_row(ids))
return torch.tensor(hist, dtype=torch.int32, device=runner.device)
return async_h2d(hist, dtype=torch.int32, device=runner.device)
def _fold_batch(*, batch, runner, rows) -> FoldedExtend:
@@ -200,16 +203,16 @@ def _fold_batch(*, batch, runner, rows) -> FoldedExtend:
pre = [p - r for p, r in zip(batch.prefix_lens, replay)]
new_ext = [x + r for x, r in zip(ext, replay)]
req_to_token = runner.req_to_token_pool.req_to_token
slots = batch.req_pool_indices_cpu.tolist()
ids, locs, keep, cpu_ids = [], [], [], []
off = fold_off = 0
for i, req in enumerate(batch.reqs):
r, n = replay[i], ext[i]
if r:
head = list(req.full_untruncated_fill_ids[pre[i] : pre[i] + r])
ids.append(torch.tensor(head, dtype=batch.input_ids.dtype, device=device))
locs.append(
req_to_token[batch.req_pool_indices[i], pre[i] : pre[i] + r].long()
)
head_ids = async_h2d(head, dtype=batch.input_ids.dtype, device=device)
ids.append(head_ids)
locs.append(req_to_token[slots[i], pre[i] : pre[i] + r].long())
cpu_ids.append(torch.tensor(head, dtype=torch.int64))
ids.append(batch.input_ids[off : off + n])
locs.append(batch.out_cache_loc[off : off + n].long())
@@ -245,16 +248,25 @@ def _fold_batch(*, batch, runner, rows) -> FoldedExtend:
keep_rows = torch.cat(keep)
scheduled_row = torch.full((fold_off,), -1, dtype=torch.int64)
scheduled_row[keep_rows] = torch.arange(keep_rows.shape[0])
keep_rows = _to_device(keep_rows, device)
scheduled_row = _to_device(scheduled_row, device)
row_floor = _to_device(row_floor, device)
return FoldedExtend(
batch=folded,
keep_rows=keep_rows.to(device, non_blocking=True),
scheduled_row=scheduled_row.to(device, non_blocking=True),
row_floor=row_floor.to(device, non_blocking=True),
keep_rows=keep_rows,
scheduled_row=scheduled_row,
row_floor=row_floor,
replay_lens=replay,
num_rows=fold_off,
)
def _to_device(t: torch.Tensor, device) -> torch.Tensor:
# A pageable host tensor's copy waits for the stream even with non_blocking.
pin = is_pin_memory_available(device)
return (t.pin_memory() if pin else t).to(device, non_blocking=pin)
def _set_execution_counts(*, batch, scheduled) -> None:
"""MLP-sync counts of a batch copied from `scheduled` with different rows."""
if scheduled.global_num_tokens is None: