[Perf] speculative: Fuse draft write-window gathering and KV translation (#44972)

This commit is contained in:
Cheng Wan
2026-10-18 05:41:34 -07:00
committed by GitHub
parent 2a83471676
commit 2e9b2cacda
8 changed files with 365 additions and 49 deletions
@@ -16,6 +16,7 @@ from sglang.kernels.spec import (
# (module, public_fn) migrated from speculative/triton_ops.
_TRITON_KERNELS = [
("cache_locs", "assign_extend_cache_locs_func"),
("cache_locs", "assign_extend_cache_locs_uniform_func"),
("cache_locs", "generate_draft_decode_kv_indices"),
("cache_locs", "generate_draft_decode_window_kv_indices"),
("eagle", "fill_bonus_tokens"),
@@ -1,6 +1,7 @@
from __future__ import annotations
from collections.abc import Callable
from typing import Optional
import torch
import triton
@@ -619,6 +620,13 @@ def assign_extend_cache_locs_uniform(
out_cache_loc,
pool_len: tl.constexpr,
draft_token_num: tl.constexpr,
full_v2p=None,
swa_v2p=None,
full_out=None,
swa_out=None,
PAGE_SIZE: tl.constexpr = 1,
num_v_pages=0,
num_swa_pages=0,
):
"""Uniform-length variant of assign_extend_cache_locs: every row extends
exactly draft_token_num tokens, so the end offset is start +
@@ -637,6 +645,21 @@ def assign_extend_cache_locs_uniform(
mask = o < draft_token_num
data = tl.load(token_pool + kv_start + o, mask=mask)
tl.store(out_cache_ptr + o, data, mask=mask)
if full_v2p is not None:
page = data // PAGE_SIZE
valid = mask & (data >= 0) & (page < num_v_pages)
physical = tl.load(
full_v2p + tl.where(valid, page, 0), mask=valid, other=-1
)
full_loc = tl.maximum(physical * PAGE_SIZE + data % PAGE_SIZE, 0)
tl.store(full_out + pid * draft_token_num + o, full_loc, mask=mask)
if swa_v2p is not None:
swa_valid = mask & (data >= 0) & (page < num_swa_pages)
physical = tl.load(
swa_v2p + tl.where(swa_valid, page, 0), mask=swa_valid, other=-1
)
swa_loc = tl.maximum(physical * PAGE_SIZE + data % PAGE_SIZE, 0)
tl.store(swa_out + pid * draft_token_num + o, swa_loc, mask=mask)
def assign_extend_cache_locs_uniform_func(
@@ -646,11 +669,30 @@ def assign_extend_cache_locs_uniform_func(
batch_size: int,
draft_token_num: int,
device,
*,
full_v2p: Optional[torch.Tensor] = None,
swa_v2p: Optional[torch.Tensor] = None,
full_out: Optional[torch.Tensor] = None,
swa_out: Optional[torch.Tensor] = None,
page_size: int = 1,
) -> torch.Tensor:
"""assign_extend_cache_locs for the uniform case (all rows extend exactly
draft_token_num tokens, e.g. spec target-verify prep). Computes end
offsets inside the kernel, removing the eager `seq_lens + draft_token_num`
add from the host critical path."""
add from the host critical path. Optional v2p tables fill physical FULL
and SWA write windows in the same launch; the returned ids stay virtual.
Mapping tables and output buffers must be contiguous, with one output
per gathered token. Translation is rank-local (no DCP widening).
"""
if full_v2p is not None:
assert full_out is not None and full_out.numel() == batch_size * draft_token_num
assert full_v2p.is_contiguous() and full_out.is_contiguous()
assert swa_v2p is None or (
swa_out is not None
and swa_out.numel() == full_out.numel()
and swa_out.is_contiguous()
and swa_v2p.is_contiguous()
)
if _is_cuda or _is_hip or _is_musa or _is_xpu:
out_cache_loc = torch.empty(
(batch_size * draft_token_num,),
@@ -664,10 +706,18 @@ def assign_extend_cache_locs_uniform_func(
out_cache_loc,
req_to_token.shape[1],
draft_token_num,
full_v2p=full_v2p,
swa_v2p=swa_v2p,
full_out=full_out,
swa_out=swa_out,
PAGE_SIZE=page_size,
num_v_pages=0 if full_v2p is None else full_v2p.numel(),
num_swa_pages=0 if swa_v2p is None else swa_v2p.numel(),
)
return out_cache_loc
# NPU / CPU platforms: fall back to the end_offset-tensor path.
assert full_v2p is None, "fused write translation requires a Triton device"
return assign_extend_cache_locs_func(
req_pool_indices=req_pool_indices,
req_to_token=req_to_token,
@@ -25,8 +25,9 @@ The two coincide on a plain pool, so nothing here does any work there.
A pool can hold several sub-pools (full attention, sliding window), each with
its own physical ids and named by an `IdSpace` (`space`). Writes are translated
once per sub-pool, by the iteration's `KVLocPlan`. A read is translated exactly
once, by the readers below and nowhere else; a backend never translates. The
once per sub-pool, by the iteration's `KVLocPlan` or its fused producer.
A read is translated exactly once, by the readers below or a captured gather
using the `IdSpace`'s live mapping. The
plan's read table exists only once a table reader asks for it (`read_table`,
`copy_page_table`), built into that reader's own capture-stable table when it
is the first; later readers take it, earlier ones translate in their own
@@ -276,12 +277,15 @@ class KVIndexTranslator:
write_virtual: Optional[torch.Tensor],
read_extent: int = 0,
write_slots: Optional[torch.Tensor] = None,
translated_writes: Optional[Dict[IdSpaceKind, torch.Tensor]] = None,
) -> KVLocPlan:
"""This iteration's ids, translated once. ``write_virtual`` is the
iteration's write window (``[batch, window]`` flattened when its
forwards write columns of it); ``read_extent`` is how far past
``seq_lens`` the iteration's reads reach. ``write_slots`` replaces
``write_virtual`` for a runner's own buffer (`bind_runner_slots`)."""
``write_virtual`` for a runner's own buffer (`bind_runner_slots`).
``translated_writes`` supplies physical outputs already computed by
the producer, keyed by this source's sub-pools."""
return KVLocPlan(
source=self,
req_pool_indices=req_pool_indices,
@@ -290,6 +294,61 @@ class KVIndexTranslator:
write_virtual=write_virtual,
read_extent=read_extent,
write_slots=write_slots,
translated_writes=translated_writes,
)
def plan_uniform_extend(
self,
*,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
seq_lens_cpu: Optional[torch.Tensor],
width: int,
) -> KVLocPlan:
"""Gather one fixed-width speculative window, translating at its producer.
The virtual window stays available for accept/reclaim bookkeeping;
draft, target and direct writers share the physical outputs. DCP and
non-CUDA devices retain the plan's ordinary write translation.
"""
from sglang.kernels.ops.speculative.cache_locs import (
assign_extend_cache_locs_uniform_func,
)
translated = None
mappings = {}
if req_pool_indices.is_cuda and self.reads_are_translated:
translated = {}
for kind in (IdSpaceKind.FULL, IdSpaceKind.SLIDING_WINDOW):
space = self.space(kind)
if space is not None and space.read_v2p is not None:
mappings[kind] = space.read_v2p
translated[kind] = torch.empty(
req_pool_indices.numel() * width,
dtype=torch.int64,
device=req_pool_indices.device,
)
writes = translated or {}
virtual = assign_extend_cache_locs_uniform_func(
req_pool_indices=req_pool_indices,
req_to_token=self.req_to_token,
start_offset=seq_lens,
batch_size=req_pool_indices.numel(),
draft_token_num=width,
device=req_pool_indices.device,
full_v2p=mappings.get(IdSpaceKind.FULL),
swa_v2p=mappings.get(IdSpaceKind.SLIDING_WINDOW),
full_out=writes.get(IdSpaceKind.FULL),
swa_out=writes.get(IdSpaceKind.SLIDING_WINDOW),
page_size=self.page_size,
)
return self.plan(
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
seq_lens_cpu=seq_lens_cpu,
write_virtual=virtual,
read_extent=width,
translated_writes=translated,
)
def own_plan(self, forward_batch, *, runner_slots: bool = False) -> KVLocPlan:
+19 -26
View File
@@ -22,7 +22,8 @@ forwards are in flight. So:
* The write window is translated once per sub-pool, by the plan: the
full-attention ids when the plan is built, another sub-pool's on first
use (`write_ids`). Every writer binds them.
use (`write_ids`), or both by the producer's fused gather. Every writer
binds them.
* A read is translated exactly once, only by the translator's read
primitives (`KVIndexTranslator`); a consumer never translates.
* The read table (`read_table`) exists only once a table reader asks for
@@ -130,6 +131,7 @@ class KVLocPlan:
write_virtual: Optional[torch.Tensor],
read_extent: int = 0,
write_slots: Optional[torch.Tensor] = None,
translated_writes: Optional[Dict[IdSpaceKind, torch.Tensor]] = None,
):
self._source = source
self.req_pool_indices = req_pool_indices
@@ -139,34 +141,25 @@ class KVLocPlan:
self.seq_lens_cpu = seq_lens_cpu
self.read_extent = read_extent
full = source.space(IdSpaceKind.FULL)
# Scheduler accept/reclaim bookkeeping still names these virtual ids.
self.write_virtual = write_virtual
self.write_physical = write_slots
self._full_key = full.key
self._write_ids: Dict[Hashable, Optional[torch.Tensor]] = {}
if write_slots is not None:
# A runner's own write buffer (graph capture, warmup): already in
# its pool's ids, naming the sink until a replay fills it, and kept
# by address by a captured graph.
# A runner's capture-stable buffer is already in its pool's ids.
# Without virtual ids, other sub-pools use sink slots at capture.
assert write_virtual is None
self.write_virtual = None if full.write is not None else write_slots
self.write_physical = write_slots
else:
# Aliases the ScheduleBatch's tensor, which stays virtual for the
# radix tree, the accept path and lazy compaction's in-flight write
# set.
self.write_virtual = write_virtual
# FIXME: a speculative iteration builds its plan before its draft
# runs, so this launch sits on the host path ahead of the draft's
# graph, idle GPU time at low concurrency. Translate the window
# inside the iteration's first captured forward instead, into a
# capture-stable buffer the plan owns, which every later forward
# and direct writer reads -- once per sub-pool, the sliding-window
# ids (derived in `write_ids`) included.
self.write_physical = (
full.write(write_virtual)
if full.write is not None and write_virtual is not None
else write_virtual
)
self._full_key = full.key
self._write_ids: Dict[Hashable, Optional[torch.Tensor]] = {
full.key: self.write_physical
}
self._write_ids[full.key] = write_slots
if translated_writes is not None:
assert write_slots is None and write_virtual is not None
for kind, ids in translated_writes.items():
space = source.space(kind)
assert space is not None and space.write is not None
assert ids.shape == write_virtual.shape
self._write_ids[space.key] = ids
self.write_physical = self.write_ids(source)
self._read_tables: Dict[Hashable, KVIndexTable] = {}
# -- writes ----------------------------------------------------------------
@@ -6,7 +6,6 @@ from typing import Optional, Union
import msgspec
import torch
from sglang.kernels.ops.speculative.cache_locs import assign_extend_cache_locs_func
from sglang.kernels.ops.speculative.dspark.dspark_schedule import (
ScheduleVerifyLensTopk,
compute_sort_survival,
@@ -796,27 +795,18 @@ def alloc_verify_window(
prefix_lens = batch.seq_lens
verify_w = verify_num_draft_tokens
positions_2d = prefix_lens.unsqueeze(1) + block_pos_offsets
verify_cache_loc = assign_extend_cache_locs_func(
plan = model_runner.kv_index_translator.plan_uniform_extend(
req_pool_indices=batch.req_pool_indices,
req_to_token=model_runner.req_to_token_pool.req_to_token,
start_offset=prefix_lens,
end_offset=prefix_lens + verify_w,
batch_size=bs,
draft_token_num=verify_w,
device=device,
seq_lens=prefix_lens,
seq_lens_cpu=seq_lens_cpu,
width=verify_w,
)
verify_cache_loc_2d = verify_cache_loc.view(bs, verify_w)
verify_cache_loc = plan.write_virtual
return VerifyWindow(
positions_2d=positions_2d,
verify_cache_loc=verify_cache_loc,
verify_cache_loc_2d=verify_cache_loc_2d,
kv_loc_plan=model_runner.kv_index_translator.plan(
req_pool_indices=batch.req_pool_indices,
seq_lens=prefix_lens,
seq_lens_cpu=seq_lens_cpu,
write_virtual=verify_cache_loc,
read_extent=verify_w,
),
verify_cache_loc_2d=verify_cache_loc.view(bs, verify_w),
kv_loc_plan=plan,
)