[Perf] speculative: Fuse draft write-window gathering and KV translation (#44972)

This commit is contained in:
Cheng Wan
2026-10-18 05:41:34 -07:00
committed by GitHub
parent 2a83471676
commit 2e9b2cacda
8 changed files with 365 additions and 49 deletions
@@ -16,6 +16,7 @@ from sglang.kernels.spec import (
# (module, public_fn) migrated from speculative/triton_ops.
_TRITON_KERNELS = [
("cache_locs", "assign_extend_cache_locs_func"),
("cache_locs", "assign_extend_cache_locs_uniform_func"),
("cache_locs", "generate_draft_decode_kv_indices"),
("cache_locs", "generate_draft_decode_window_kv_indices"),
("eagle", "fill_bonus_tokens"),
@@ -1,6 +1,7 @@
from __future__ import annotations
from collections.abc import Callable
from typing import Optional
import torch
import triton
@@ -619,6 +620,13 @@ def assign_extend_cache_locs_uniform(
out_cache_loc,
pool_len: tl.constexpr,
draft_token_num: tl.constexpr,
full_v2p=None,
swa_v2p=None,
full_out=None,
swa_out=None,
PAGE_SIZE: tl.constexpr = 1,
num_v_pages=0,
num_swa_pages=0,
):
"""Uniform-length variant of assign_extend_cache_locs: every row extends
exactly draft_token_num tokens, so the end offset is start +
@@ -637,6 +645,21 @@ def assign_extend_cache_locs_uniform(
mask = o < draft_token_num
data = tl.load(token_pool + kv_start + o, mask=mask)
tl.store(out_cache_ptr + o, data, mask=mask)
if full_v2p is not None:
page = data // PAGE_SIZE
valid = mask & (data >= 0) & (page < num_v_pages)
physical = tl.load(
full_v2p + tl.where(valid, page, 0), mask=valid, other=-1
)
full_loc = tl.maximum(physical * PAGE_SIZE + data % PAGE_SIZE, 0)
tl.store(full_out + pid * draft_token_num + o, full_loc, mask=mask)
if swa_v2p is not None:
swa_valid = mask & (data >= 0) & (page < num_swa_pages)
physical = tl.load(
swa_v2p + tl.where(swa_valid, page, 0), mask=swa_valid, other=-1
)
swa_loc = tl.maximum(physical * PAGE_SIZE + data % PAGE_SIZE, 0)
tl.store(swa_out + pid * draft_token_num + o, swa_loc, mask=mask)
def assign_extend_cache_locs_uniform_func(
@@ -646,11 +669,30 @@ def assign_extend_cache_locs_uniform_func(
batch_size: int,
draft_token_num: int,
device,
*,
full_v2p: Optional[torch.Tensor] = None,
swa_v2p: Optional[torch.Tensor] = None,
full_out: Optional[torch.Tensor] = None,
swa_out: Optional[torch.Tensor] = None,
page_size: int = 1,
) -> torch.Tensor:
"""assign_extend_cache_locs for the uniform case (all rows extend exactly
draft_token_num tokens, e.g. spec target-verify prep). Computes end
offsets inside the kernel, removing the eager `seq_lens + draft_token_num`
add from the host critical path."""
add from the host critical path. Optional v2p tables fill physical FULL
and SWA write windows in the same launch; the returned ids stay virtual.
Mapping tables and output buffers must be contiguous, with one output
per gathered token. Translation is rank-local (no DCP widening).
"""
if full_v2p is not None:
assert full_out is not None and full_out.numel() == batch_size * draft_token_num
assert full_v2p.is_contiguous() and full_out.is_contiguous()
assert swa_v2p is None or (
swa_out is not None
and swa_out.numel() == full_out.numel()
and swa_out.is_contiguous()
and swa_v2p.is_contiguous()
)
if _is_cuda or _is_hip or _is_musa or _is_xpu:
out_cache_loc = torch.empty(
(batch_size * draft_token_num,),
@@ -664,10 +706,18 @@ def assign_extend_cache_locs_uniform_func(
out_cache_loc,
req_to_token.shape[1],
draft_token_num,
full_v2p=full_v2p,
swa_v2p=swa_v2p,
full_out=full_out,
swa_out=swa_out,
PAGE_SIZE=page_size,
num_v_pages=0 if full_v2p is None else full_v2p.numel(),
num_swa_pages=0 if swa_v2p is None else swa_v2p.numel(),
)
return out_cache_loc
# NPU / CPU platforms: fall back to the end_offset-tensor path.
assert full_v2p is None, "fused write translation requires a Triton device"
return assign_extend_cache_locs_func(
req_pool_indices=req_pool_indices,
req_to_token=req_to_token,
@@ -25,8 +25,9 @@ The two coincide on a plain pool, so nothing here does any work there.
A pool can hold several sub-pools (full attention, sliding window), each with
its own physical ids and named by an `IdSpace` (`space`). Writes are translated
once per sub-pool, by the iteration's `KVLocPlan`. A read is translated exactly
once, by the readers below and nowhere else; a backend never translates. The
once per sub-pool, by the iteration's `KVLocPlan` or its fused producer.
A read is translated exactly once, by the readers below or a captured gather
using the `IdSpace`'s live mapping. The
plan's read table exists only once a table reader asks for it (`read_table`,
`copy_page_table`), built into that reader's own capture-stable table when it
is the first; later readers take it, earlier ones translate in their own
@@ -276,12 +277,15 @@ class KVIndexTranslator:
write_virtual: Optional[torch.Tensor],
read_extent: int = 0,
write_slots: Optional[torch.Tensor] = None,
translated_writes: Optional[Dict[IdSpaceKind, torch.Tensor]] = None,
) -> KVLocPlan:
"""This iteration's ids, translated once. ``write_virtual`` is the
iteration's write window (``[batch, window]`` flattened when its
forwards write columns of it); ``read_extent`` is how far past
``seq_lens`` the iteration's reads reach. ``write_slots`` replaces
``write_virtual`` for a runner's own buffer (`bind_runner_slots`)."""
``write_virtual`` for a runner's own buffer (`bind_runner_slots`).
``translated_writes`` supplies physical outputs already computed by
the producer, keyed by this source's sub-pools."""
return KVLocPlan(
source=self,
req_pool_indices=req_pool_indices,
@@ -290,6 +294,61 @@ class KVIndexTranslator:
write_virtual=write_virtual,
read_extent=read_extent,
write_slots=write_slots,
translated_writes=translated_writes,
)
def plan_uniform_extend(
self,
*,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
seq_lens_cpu: Optional[torch.Tensor],
width: int,
) -> KVLocPlan:
"""Gather one fixed-width speculative window, translating at its producer.
The virtual window stays available for accept/reclaim bookkeeping;
draft, target and direct writers share the physical outputs. DCP and
non-CUDA devices retain the plan's ordinary write translation.
"""
from sglang.kernels.ops.speculative.cache_locs import (
assign_extend_cache_locs_uniform_func,
)
translated = None
mappings = {}
if req_pool_indices.is_cuda and self.reads_are_translated:
translated = {}
for kind in (IdSpaceKind.FULL, IdSpaceKind.SLIDING_WINDOW):
space = self.space(kind)
if space is not None and space.read_v2p is not None:
mappings[kind] = space.read_v2p
translated[kind] = torch.empty(
req_pool_indices.numel() * width,
dtype=torch.int64,
device=req_pool_indices.device,
)
writes = translated or {}
virtual = assign_extend_cache_locs_uniform_func(
req_pool_indices=req_pool_indices,
req_to_token=self.req_to_token,
start_offset=seq_lens,
batch_size=req_pool_indices.numel(),
draft_token_num=width,
device=req_pool_indices.device,
full_v2p=mappings.get(IdSpaceKind.FULL),
swa_v2p=mappings.get(IdSpaceKind.SLIDING_WINDOW),
full_out=writes.get(IdSpaceKind.FULL),
swa_out=writes.get(IdSpaceKind.SLIDING_WINDOW),
page_size=self.page_size,
)
return self.plan(
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
seq_lens_cpu=seq_lens_cpu,
write_virtual=virtual,
read_extent=width,
translated_writes=translated,
)
def own_plan(self, forward_batch, *, runner_slots: bool = False) -> KVLocPlan:
+19 -26
View File
@@ -22,7 +22,8 @@ forwards are in flight. So:
* The write window is translated once per sub-pool, by the plan: the
full-attention ids when the plan is built, another sub-pool's on first
use (`write_ids`). Every writer binds them.
use (`write_ids`), or both by the producer's fused gather. Every writer
binds them.
* A read is translated exactly once, only by the translator's read
primitives (`KVIndexTranslator`); a consumer never translates.
* The read table (`read_table`) exists only once a table reader asks for
@@ -130,6 +131,7 @@ class KVLocPlan:
write_virtual: Optional[torch.Tensor],
read_extent: int = 0,
write_slots: Optional[torch.Tensor] = None,
translated_writes: Optional[Dict[IdSpaceKind, torch.Tensor]] = None,
):
self._source = source
self.req_pool_indices = req_pool_indices
@@ -139,34 +141,25 @@ class KVLocPlan:
self.seq_lens_cpu = seq_lens_cpu
self.read_extent = read_extent
full = source.space(IdSpaceKind.FULL)
# Scheduler accept/reclaim bookkeeping still names these virtual ids.
self.write_virtual = write_virtual
self.write_physical = write_slots
self._full_key = full.key
self._write_ids: Dict[Hashable, Optional[torch.Tensor]] = {}
if write_slots is not None:
# A runner's own write buffer (graph capture, warmup): already in
# its pool's ids, naming the sink until a replay fills it, and kept
# by address by a captured graph.
# A runner's capture-stable buffer is already in its pool's ids.
# Without virtual ids, other sub-pools use sink slots at capture.
assert write_virtual is None
self.write_virtual = None if full.write is not None else write_slots
self.write_physical = write_slots
else:
# Aliases the ScheduleBatch's tensor, which stays virtual for the
# radix tree, the accept path and lazy compaction's in-flight write
# set.
self.write_virtual = write_virtual
# FIXME: a speculative iteration builds its plan before its draft
# runs, so this launch sits on the host path ahead of the draft's
# graph, idle GPU time at low concurrency. Translate the window
# inside the iteration's first captured forward instead, into a
# capture-stable buffer the plan owns, which every later forward
# and direct writer reads -- once per sub-pool, the sliding-window
# ids (derived in `write_ids`) included.
self.write_physical = (
full.write(write_virtual)
if full.write is not None and write_virtual is not None
else write_virtual
)
self._full_key = full.key
self._write_ids: Dict[Hashable, Optional[torch.Tensor]] = {
full.key: self.write_physical
}
self._write_ids[full.key] = write_slots
if translated_writes is not None:
assert write_slots is None and write_virtual is not None
for kind, ids in translated_writes.items():
space = source.space(kind)
assert space is not None and space.write is not None
assert ids.shape == write_virtual.shape
self._write_ids[space.key] = ids
self.write_physical = self.write_ids(source)
self._read_tables: Dict[Hashable, KVIndexTable] = {}
# -- writes ----------------------------------------------------------------
@@ -6,7 +6,6 @@ from typing import Optional, Union
import msgspec
import torch
from sglang.kernels.ops.speculative.cache_locs import assign_extend_cache_locs_func
from sglang.kernels.ops.speculative.dspark.dspark_schedule import (
ScheduleVerifyLensTopk,
compute_sort_survival,
@@ -796,27 +795,18 @@ def alloc_verify_window(
prefix_lens = batch.seq_lens
verify_w = verify_num_draft_tokens
positions_2d = prefix_lens.unsqueeze(1) + block_pos_offsets
verify_cache_loc = assign_extend_cache_locs_func(
plan = model_runner.kv_index_translator.plan_uniform_extend(
req_pool_indices=batch.req_pool_indices,
req_to_token=model_runner.req_to_token_pool.req_to_token,
start_offset=prefix_lens,
end_offset=prefix_lens + verify_w,
batch_size=bs,
draft_token_num=verify_w,
device=device,
seq_lens=prefix_lens,
seq_lens_cpu=seq_lens_cpu,
width=verify_w,
)
verify_cache_loc_2d = verify_cache_loc.view(bs, verify_w)
verify_cache_loc = plan.write_virtual
return VerifyWindow(
positions_2d=positions_2d,
verify_cache_loc=verify_cache_loc,
verify_cache_loc_2d=verify_cache_loc_2d,
kv_loc_plan=model_runner.kv_index_translator.plan(
req_pool_indices=batch.req_pool_indices,
seq_lens=prefix_lens,
seq_lens_cpu=seq_lens_cpu,
write_virtual=verify_cache_loc,
read_extent=verify_w,
),
verify_cache_loc_2d=verify_cache_loc.view(bs, verify_w),
kv_loc_plan=plan,
)
@@ -0,0 +1,78 @@
"""Fused write gathering must reuse kernels across pool sizes and sink bad IDs."""
import unittest
import torch
from sglang.kernels.ops.speculative.cache_locs import (
assign_extend_cache_locs_uniform,
assign_extend_cache_locs_uniform_func,
)
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="1-gpu-large")
class TestUniformExtendTranslation(CustomTestCase):
def test_pool_sizes_are_runtime_bounds(self):
"""Different virtual table lengths must share the warmed kernel.
Negative, tombstoned and past-table slots map to the sink independently
in FULL and SWA, including when their virtual tables differ in size.
"""
ps = 128
tokens = [
-129,
-1,
0,
3 * ps + 7,
62 * ps + 1,
126 * ps + 2,
254 * ps + 3,
511 * ps,
]
req_to_token = torch.tensor([tokens], device="cuda", dtype=torch.int64)
rows = torch.zeros(1, device="cuda", dtype=torch.int64)
starts = torch.zeros_like(rows)
full_out = torch.empty(8, device="cuda", dtype=torch.int64)
swa_out = torch.empty_like(full_out)
cache = assign_extend_cache_locs_uniform.device_caches[
torch.cuda.current_device()
][0]
saved = dict(cache)
cache.clear()
try:
for full_size, swa_size in ((127, 63), (255, 127), (511, 255), (127, 511)):
full = torch.arange(full_size, device="cuda", dtype=torch.int64)
swa = torch.arange(swa_size, device="cuda", dtype=torch.int64) + 1
full[0] = swa[0] = 0
full[3] = swa[3] = -1
got_virtual = assign_extend_cache_locs_uniform_func(
rows,
req_to_token,
starts,
1,
8,
"cuda",
full_v2p=full,
swa_v2p=swa,
full_out=full_out,
swa_out=swa_out,
page_size=ps,
)
torch.testing.assert_close(got_virtual, req_to_token.flatten())
for mapping, output in ((full, full_out), (swa, swa_out)):
table = mapping.cpu().tolist()
expected = []
for token in tokens:
page, offset = divmod(token, ps)
physical = table[page] if 0 <= page < len(table) else -1
expected.append(max(physical * ps + offset, 0))
torch.testing.assert_close(output, output.new_tensor(expected))
self.assertEqual(len(cache), 1, (full_size, swa_size))
finally:
cache.update(saved)
if __name__ == "__main__":
unittest.main()
@@ -55,8 +55,9 @@ from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.runtime_context import publish, reset_context
from sglang.srt.server_args import ServerArgs
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=8, suite="base-a-test-cpu")
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
_DEV = "cpu"
_PS = 2
@@ -126,7 +127,7 @@ class _FakeUnifiedSWAKVPool:
self._swa_allocator = swa_allocator
class TestKVLocPlan(unittest.TestCase):
class TestKVLocPlan(CustomTestCase):
def setUp(self):
# `KVIndexTranslator.__init__` reads `attn_dcp_size`, a derived
# parallel width that only exists once a config is published.
@@ -229,6 +230,29 @@ class TestKVLocPlan(unittest.TestCase):
self.assertIs(plan.write_ids(self.private_draft), self.window)
self.assertTrue(torch.equal(self.window, window)) # never mutated
def test_precomputed_writes_seed_each_space_independently(self):
expected = {
kind: self.target.space(kind).write(self.window) for kind in (_FULL, _SWA)
}
for kinds in ((_FULL,), (_SWA,), (_FULL, _SWA)):
with self.subTest(precomputed=kinds):
supplied = {kind: expected[kind] for kind in kinds}
plan = self.target.plan(
req_pool_indices=self.rpi,
seq_lens=self.seq_lens,
seq_lens_cpu=self.seq_lens.clone(),
write_virtual=self.window,
translated_writes=supplied,
)
for kind in (_FULL, _SWA):
ids = plan.write_ids(self.target, kind=kind)
torch.testing.assert_close(ids, expected[kind])
self.assertIs(plan.write_ids(self.target, kind=kind), ids)
if kind in supplied:
self.assertIs(ids, supplied[kind])
self.assertIs(plan.write_ids(self.fused_draft), plan.write_physical)
self.assertIs(plan.write_ids(self.private_draft), self.window)
def test_cols_slice_names_a_split_batch_s_tokens(self):
"""Two-batch overlap splits a forward's tokens; each half writes its
own tokens of the window, whatever the forward's own columns were."""
@@ -0,0 +1,121 @@
"""GPU producer/plan contract: virtual bookkeeping and shared physical writes."""
import unittest
from types import SimpleNamespace
import torch
from sglang.kernels.ops.memory.virtual_slot import translate_token_ids
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
from sglang.srt.mem_cache.kv_loc_plan import IdSpace, IdSpaceKind
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=15, stage="extra-a", runner_config="1-gpu-small")
@unittest.skipUnless(torch.cuda.is_available(), "requires CUDA")
class TestUniformExtendPlan(CustomTestCase):
def test_shared_writes_preserve_virtual_ids_and_page_offsets(self):
"""A fused gather must retain per-page offsets and every reader's space.
Changing v2p after plan creation must not retranslate that iteration's
writes; the next iteration must pick up the new map. This catches
losing the producer's outputs or incorrectly sharing FULL with SWA.
"""
for ps in (1, 128):
for width in (7, 8, 65):
for swa_enabled in (False, True):
with self.subTest(page_size=ps, width=width, swa=swa_enabled):
self._check_plan(ps, width, swa_enabled)
def _check_plan(self, ps, width, swa_enabled):
full_kind, swa_kind = IdSpaceKind.FULL, IdSpaceKind.SLIDING_WINDOW
row_tokens = ((2 * ps + width + ps - 1) // ps) * ps
page_count = 3 * row_tokens // ps
req_to_token = torch.arange(3 * row_tokens, device="cuda").view(3, row_tokens)
req_to_token[0].zero_()
full = (
torch.arange(page_count, device="cuda", dtype=torch.int64)
.flip(0)
.contiguous()
)
full[0] = 0
swa = torch.arange(page_count, device="cuda", dtype=torch.int64)
swa[row_tokens // ps + 1] = -1
def space(key, table):
return IdSpace(
key=key,
read_v2p=table,
write=lambda ids: translate_token_ids(loc=ids, v2p=table, page_size=ps),
)
source = KVIndexTranslator.__new__(KVIndexTranslator)
source.page_size = ps
source.req_to_token = req_to_token
source._spaces = {full_kind: space("full", full)}
if swa_enabled:
source._spaces[swa_kind] = space("swa", swa)
fused = KVIndexTranslator.__new__(KVIndexTranslator)
fused._spaces = {full_kind: source.space(full_kind)}
private = KVIndexTranslator.__new__(KVIndexTranslator)
private._spaces = {full_kind: IdSpace(key="private")}
rows = torch.tensor([2, 1, 0], device="cuda")
lengths = torch.tensor([ps - 1, ps + 1, 0], device="cuda")
expected_virtual = req_to_token[
rows[:, None], lengths[:, None] + torch.arange(width, device="cuda")
].flatten()
for iteration in range(2):
if iteration:
full[1:] = 2
swa[1:] = 3
plan = source.plan_uniform_extend(
req_pool_indices=rows,
seq_lens=lengths,
seq_lens_cpu=lengths.cpu(),
width=width,
)
expected = {}
for kind, mapping in ((full_kind, full), (swa_kind, swa)):
if source.space(kind) is not None:
expected[kind] = (
mapping[expected_virtual // ps] * ps + expected_virtual % ps
).clamp_min(0)
torch.testing.assert_close(plan.write_virtual, expected_virtual)
# Retain this iteration's ids even if a later allocator update
# changes the table before another consumer asks for its view.
full[1:] = 4
swa[1:] = 5
torch.testing.assert_close(plan.write_ids(source), expected[full_kind])
self.assertIs(plan.write_ids(fused), plan.write_ids(source))
torch.testing.assert_close(plan.write_ids(private), expected_virtual)
if swa_enabled:
torch.testing.assert_close(
plan.write_ids(source, kind=swa_kind), expected[swa_kind]
)
fb = SimpleNamespace()
plan.bind(fb, fused, cols=slice(0, width - 1))
torch.testing.assert_close(
fb.out_cache_loc, expected[full_kind].view(3, width)[:, :-1].reshape(-1)
)
torch.testing.assert_close(
fb.out_cache_loc_virtual,
expected_virtual.view(3, width)[:, :-1].reshape(-1),
)
packed = torch.tensor([width + 1, -1, 0], device="cuda")
torch.testing.assert_close(
plan.write_ids(source, cols=packed),
torch.stack(
(
expected[full_kind][width + 1],
expected[full_kind].new_zeros(()),
expected[full_kind][0],
)
),
)
if __name__ == "__main__":
unittest.main()