From 5e630fa57126b7868c99e8d8efedf75f78993366 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Mon, 19 Oct 2026 19:39:39 -0700 Subject: [PATCH] [Perf] deepseek-v4.1: Drop host syncs from encoder-replay folding and tail metadata (#45275) --- .../attention/dsv4/attn_metadata_kernels.py | 6 +++- .../layers/attention/deepseek_v4_backend.py | 3 +- .../srt/model_executor/encoder_swa_replay.py | 30 +++++++++++++------ .../model_executor/test_encoder_swa_replay.py | 1 + 4 files changed, 29 insertions(+), 11 deletions(-) diff --git a/python/sglang/kernels/ops/attention/dsv4/attn_metadata_kernels.py b/python/sglang/kernels/ops/attention/dsv4/attn_metadata_kernels.py index 0f4b589ab9..41d6996e0a 100644 --- a/python/sglang/kernels/ops/attention/dsv4/attn_metadata_kernels.py +++ b/python/sglang/kernels/ops/attention/dsv4/attn_metadata_kernels.py @@ -7,6 +7,8 @@ import torch import triton import triton.language as tl +from sglang.srt.utils.common import async_h2d + def _inputs_on_cuda(*args, **kwargs) -> bool: """Route kernel dispatch by input placement: the first tensor argument @@ -480,7 +482,9 @@ def late_layer_tail_layout( floor = torch.full((t,), max(0, s - tail_len), dtype=torch.int32, device=device) return torch.arange(n - t, n, device=device), tail_lens_cpu, floor # One H2D copy for the three length vectors; launch count does not grow with bs. - lens = torch.tensor([extend_lens_cpu, tail_lens_cpu, seq_lens_cpu], device=device) + lens = async_h2d( + [extend_lens_cpu, tail_lens_cpu, seq_lens_cpu], dtype=torch.int64, device=device + ) extend_lens, tail_lens, seq_lens = lens[0], lens[1], lens[2] total = sum(tail_lens_cpu) req = torch.repeat_interleave( diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index e00ea4a681..4ebb31c221 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -1836,7 +1836,7 @@ class DeepseekV4AttnBackend( token_indices=token_indices, contiguous_start=contiguous_start, ) - tail_lens = torch.tensor(tail_lens_cpu, dtype=torch.int32, device=device) + tail_lens = async_h2d(tail_lens_cpu, dtype=torch.int32, device=device) cp_tail = ( self._late_layer_tail_cp_layout(forward_batch, token_indices, tail_lens) if is_cp_active(forward_batch) @@ -1876,6 +1876,7 @@ class DeepseekV4AttnBackend( metadata.low_ratio_req_indices = torch.repeat_interleave( forward_batch.req_pool_indices.to(torch.int64), tail_lens.to(torch.int64), + output_size=sum(tail_lens_cpu), ) positions = _tail_rows( forward_batch.positions, diff --git a/python/sglang/srt/model_executor/encoder_swa_replay.py b/python/sglang/srt/model_executor/encoder_swa_replay.py index f5529abb1b..d904378af4 100644 --- a/python/sglang/srt/model_executor/encoder_swa_replay.py +++ b/python/sglang/srt/model_executor/encoder_swa_replay.py @@ -4,6 +4,8 @@ from typing import Any, Optional import msgspec import torch +from sglang.srt.utils.common import async_h2d, is_pin_memory_available + class FoldedExtend(msgspec.Struct, frozen=True): """An extend batch whose prefix hits start at their replay start.""" @@ -65,7 +67,8 @@ def drop_folded_rows( # Decoder-SWA tail rows index the folded extend; keep those on original rows. keep = _tail_rows_on_original( folded=folded, tail_lens=tail_lens, num_tail_rows=token_indices.shape[0] - ).to(token_indices.device, non_blocking=True) + ) + keep = _to_device(keep, token_indices.device) logits_output.hidden_states_token_indices = folded.scheduled_row[ token_indices[keep] ] @@ -187,7 +190,7 @@ def _engram_history(*, reqs, starts, runner): for r, s in zip(reqs, starts): ids = r.full_untruncated_fill_ids[max(0, s - n) : s] hist.append(hasher.history_row(ids)) - return torch.tensor(hist, dtype=torch.int32, device=runner.device) + return async_h2d(hist, dtype=torch.int32, device=runner.device) def _fold_batch(*, batch, runner, rows) -> FoldedExtend: @@ -200,16 +203,16 @@ def _fold_batch(*, batch, runner, rows) -> FoldedExtend: pre = [p - r for p, r in zip(batch.prefix_lens, replay)] new_ext = [x + r for x, r in zip(ext, replay)] req_to_token = runner.req_to_token_pool.req_to_token + slots = batch.req_pool_indices_cpu.tolist() ids, locs, keep, cpu_ids = [], [], [], [] off = fold_off = 0 for i, req in enumerate(batch.reqs): r, n = replay[i], ext[i] if r: head = list(req.full_untruncated_fill_ids[pre[i] : pre[i] + r]) - ids.append(torch.tensor(head, dtype=batch.input_ids.dtype, device=device)) - locs.append( - req_to_token[batch.req_pool_indices[i], pre[i] : pre[i] + r].long() - ) + head_ids = async_h2d(head, dtype=batch.input_ids.dtype, device=device) + ids.append(head_ids) + locs.append(req_to_token[slots[i], pre[i] : pre[i] + r].long()) cpu_ids.append(torch.tensor(head, dtype=torch.int64)) ids.append(batch.input_ids[off : off + n]) locs.append(batch.out_cache_loc[off : off + n].long()) @@ -245,16 +248,25 @@ def _fold_batch(*, batch, runner, rows) -> FoldedExtend: keep_rows = torch.cat(keep) scheduled_row = torch.full((fold_off,), -1, dtype=torch.int64) scheduled_row[keep_rows] = torch.arange(keep_rows.shape[0]) + keep_rows = _to_device(keep_rows, device) + scheduled_row = _to_device(scheduled_row, device) + row_floor = _to_device(row_floor, device) return FoldedExtend( batch=folded, - keep_rows=keep_rows.to(device, non_blocking=True), - scheduled_row=scheduled_row.to(device, non_blocking=True), - row_floor=row_floor.to(device, non_blocking=True), + keep_rows=keep_rows, + scheduled_row=scheduled_row, + row_floor=row_floor, replay_lens=replay, num_rows=fold_off, ) +def _to_device(t: torch.Tensor, device) -> torch.Tensor: + # A pageable host tensor's copy waits for the stream even with non_blocking. + pin = is_pin_memory_available(device) + return (t.pin_memory() if pin else t).to(device, non_blocking=pin) + + def _set_execution_counts(*, batch, scheduled) -> None: """MLP-sync counts of a batch copied from `scheduled` with different rows.""" if scheduled.global_num_tokens is None: diff --git a/test/registered/unit/model_executor/test_encoder_swa_replay.py b/test/registered/unit/model_executor/test_encoder_swa_replay.py index ee6b10393a..9e781286b7 100644 --- a/test/registered/unit/model_executor/test_encoder_swa_replay.py +++ b/test/registered/unit/model_executor/test_encoder_swa_replay.py @@ -94,6 +94,7 @@ class _Fixture: reqs=reqs, encoder_swa_reset=[True] * len(REQS), req_pool_indices=torch.tensor([s for s, _, _ in REQS]), + req_pool_indices_cpu=torch.tensor([s for s, _, _ in REQS]), prefix_lens=[p for _, p, _ in REQS], extend_lens=[n for _, _, n in REQS], extend_num_tokens=self.extend_num_tokens,