[Perf] speculative: Fuse draft write-window gathering and KV translation (#44972)
This commit is contained in:
@@ -16,6 +16,7 @@ from sglang.kernels.spec import (
|
||||
# (module, public_fn) migrated from speculative/triton_ops.
|
||||
_TRITON_KERNELS = [
|
||||
("cache_locs", "assign_extend_cache_locs_func"),
|
||||
("cache_locs", "assign_extend_cache_locs_uniform_func"),
|
||||
("cache_locs", "generate_draft_decode_kv_indices"),
|
||||
("cache_locs", "generate_draft_decode_window_kv_indices"),
|
||||
("eagle", "fill_bonus_tokens"),
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
import triton
|
||||
@@ -619,6 +620,13 @@ def assign_extend_cache_locs_uniform(
|
||||
out_cache_loc,
|
||||
pool_len: tl.constexpr,
|
||||
draft_token_num: tl.constexpr,
|
||||
full_v2p=None,
|
||||
swa_v2p=None,
|
||||
full_out=None,
|
||||
swa_out=None,
|
||||
PAGE_SIZE: tl.constexpr = 1,
|
||||
num_v_pages=0,
|
||||
num_swa_pages=0,
|
||||
):
|
||||
"""Uniform-length variant of assign_extend_cache_locs: every row extends
|
||||
exactly draft_token_num tokens, so the end offset is start +
|
||||
@@ -637,6 +645,21 @@ def assign_extend_cache_locs_uniform(
|
||||
mask = o < draft_token_num
|
||||
data = tl.load(token_pool + kv_start + o, mask=mask)
|
||||
tl.store(out_cache_ptr + o, data, mask=mask)
|
||||
if full_v2p is not None:
|
||||
page = data // PAGE_SIZE
|
||||
valid = mask & (data >= 0) & (page < num_v_pages)
|
||||
physical = tl.load(
|
||||
full_v2p + tl.where(valid, page, 0), mask=valid, other=-1
|
||||
)
|
||||
full_loc = tl.maximum(physical * PAGE_SIZE + data % PAGE_SIZE, 0)
|
||||
tl.store(full_out + pid * draft_token_num + o, full_loc, mask=mask)
|
||||
if swa_v2p is not None:
|
||||
swa_valid = mask & (data >= 0) & (page < num_swa_pages)
|
||||
physical = tl.load(
|
||||
swa_v2p + tl.where(swa_valid, page, 0), mask=swa_valid, other=-1
|
||||
)
|
||||
swa_loc = tl.maximum(physical * PAGE_SIZE + data % PAGE_SIZE, 0)
|
||||
tl.store(swa_out + pid * draft_token_num + o, swa_loc, mask=mask)
|
||||
|
||||
|
||||
def assign_extend_cache_locs_uniform_func(
|
||||
@@ -646,11 +669,30 @@ def assign_extend_cache_locs_uniform_func(
|
||||
batch_size: int,
|
||||
draft_token_num: int,
|
||||
device,
|
||||
*,
|
||||
full_v2p: Optional[torch.Tensor] = None,
|
||||
swa_v2p: Optional[torch.Tensor] = None,
|
||||
full_out: Optional[torch.Tensor] = None,
|
||||
swa_out: Optional[torch.Tensor] = None,
|
||||
page_size: int = 1,
|
||||
) -> torch.Tensor:
|
||||
"""assign_extend_cache_locs for the uniform case (all rows extend exactly
|
||||
draft_token_num tokens, e.g. spec target-verify prep). Computes end
|
||||
offsets inside the kernel, removing the eager `seq_lens + draft_token_num`
|
||||
add from the host critical path."""
|
||||
add from the host critical path. Optional v2p tables fill physical FULL
|
||||
and SWA write windows in the same launch; the returned ids stay virtual.
|
||||
Mapping tables and output buffers must be contiguous, with one output
|
||||
per gathered token. Translation is rank-local (no DCP widening).
|
||||
"""
|
||||
if full_v2p is not None:
|
||||
assert full_out is not None and full_out.numel() == batch_size * draft_token_num
|
||||
assert full_v2p.is_contiguous() and full_out.is_contiguous()
|
||||
assert swa_v2p is None or (
|
||||
swa_out is not None
|
||||
and swa_out.numel() == full_out.numel()
|
||||
and swa_out.is_contiguous()
|
||||
and swa_v2p.is_contiguous()
|
||||
)
|
||||
if _is_cuda or _is_hip or _is_musa or _is_xpu:
|
||||
out_cache_loc = torch.empty(
|
||||
(batch_size * draft_token_num,),
|
||||
@@ -664,10 +706,18 @@ def assign_extend_cache_locs_uniform_func(
|
||||
out_cache_loc,
|
||||
req_to_token.shape[1],
|
||||
draft_token_num,
|
||||
full_v2p=full_v2p,
|
||||
swa_v2p=swa_v2p,
|
||||
full_out=full_out,
|
||||
swa_out=swa_out,
|
||||
PAGE_SIZE=page_size,
|
||||
num_v_pages=0 if full_v2p is None else full_v2p.numel(),
|
||||
num_swa_pages=0 if swa_v2p is None else swa_v2p.numel(),
|
||||
)
|
||||
return out_cache_loc
|
||||
|
||||
# NPU / CPU platforms: fall back to the end_offset-tensor path.
|
||||
assert full_v2p is None, "fused write translation requires a Triton device"
|
||||
return assign_extend_cache_locs_func(
|
||||
req_pool_indices=req_pool_indices,
|
||||
req_to_token=req_to_token,
|
||||
|
||||
@@ -25,8 +25,9 @@ The two coincide on a plain pool, so nothing here does any work there.
|
||||
|
||||
A pool can hold several sub-pools (full attention, sliding window), each with
|
||||
its own physical ids and named by an `IdSpace` (`space`). Writes are translated
|
||||
once per sub-pool, by the iteration's `KVLocPlan`. A read is translated exactly
|
||||
once, by the readers below and nowhere else; a backend never translates. The
|
||||
once per sub-pool, by the iteration's `KVLocPlan` or its fused producer.
|
||||
A read is translated exactly once, by the readers below or a captured gather
|
||||
using the `IdSpace`'s live mapping. The
|
||||
plan's read table exists only once a table reader asks for it (`read_table`,
|
||||
`copy_page_table`), built into that reader's own capture-stable table when it
|
||||
is the first; later readers take it, earlier ones translate in their own
|
||||
@@ -276,12 +277,15 @@ class KVIndexTranslator:
|
||||
write_virtual: Optional[torch.Tensor],
|
||||
read_extent: int = 0,
|
||||
write_slots: Optional[torch.Tensor] = None,
|
||||
translated_writes: Optional[Dict[IdSpaceKind, torch.Tensor]] = None,
|
||||
) -> KVLocPlan:
|
||||
"""This iteration's ids, translated once. ``write_virtual`` is the
|
||||
iteration's write window (``[batch, window]`` flattened when its
|
||||
forwards write columns of it); ``read_extent`` is how far past
|
||||
``seq_lens`` the iteration's reads reach. ``write_slots`` replaces
|
||||
``write_virtual`` for a runner's own buffer (`bind_runner_slots`)."""
|
||||
``write_virtual`` for a runner's own buffer (`bind_runner_slots`).
|
||||
``translated_writes`` supplies physical outputs already computed by
|
||||
the producer, keyed by this source's sub-pools."""
|
||||
return KVLocPlan(
|
||||
source=self,
|
||||
req_pool_indices=req_pool_indices,
|
||||
@@ -290,6 +294,61 @@ class KVIndexTranslator:
|
||||
write_virtual=write_virtual,
|
||||
read_extent=read_extent,
|
||||
write_slots=write_slots,
|
||||
translated_writes=translated_writes,
|
||||
)
|
||||
|
||||
def plan_uniform_extend(
|
||||
self,
|
||||
*,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
seq_lens_cpu: Optional[torch.Tensor],
|
||||
width: int,
|
||||
) -> KVLocPlan:
|
||||
"""Gather one fixed-width speculative window, translating at its producer.
|
||||
|
||||
The virtual window stays available for accept/reclaim bookkeeping;
|
||||
draft, target and direct writers share the physical outputs. DCP and
|
||||
non-CUDA devices retain the plan's ordinary write translation.
|
||||
"""
|
||||
from sglang.kernels.ops.speculative.cache_locs import (
|
||||
assign_extend_cache_locs_uniform_func,
|
||||
)
|
||||
|
||||
translated = None
|
||||
mappings = {}
|
||||
if req_pool_indices.is_cuda and self.reads_are_translated:
|
||||
translated = {}
|
||||
for kind in (IdSpaceKind.FULL, IdSpaceKind.SLIDING_WINDOW):
|
||||
space = self.space(kind)
|
||||
if space is not None and space.read_v2p is not None:
|
||||
mappings[kind] = space.read_v2p
|
||||
translated[kind] = torch.empty(
|
||||
req_pool_indices.numel() * width,
|
||||
dtype=torch.int64,
|
||||
device=req_pool_indices.device,
|
||||
)
|
||||
writes = translated or {}
|
||||
virtual = assign_extend_cache_locs_uniform_func(
|
||||
req_pool_indices=req_pool_indices,
|
||||
req_to_token=self.req_to_token,
|
||||
start_offset=seq_lens,
|
||||
batch_size=req_pool_indices.numel(),
|
||||
draft_token_num=width,
|
||||
device=req_pool_indices.device,
|
||||
full_v2p=mappings.get(IdSpaceKind.FULL),
|
||||
swa_v2p=mappings.get(IdSpaceKind.SLIDING_WINDOW),
|
||||
full_out=writes.get(IdSpaceKind.FULL),
|
||||
swa_out=writes.get(IdSpaceKind.SLIDING_WINDOW),
|
||||
page_size=self.page_size,
|
||||
)
|
||||
return self.plan(
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
write_virtual=virtual,
|
||||
read_extent=width,
|
||||
translated_writes=translated,
|
||||
)
|
||||
|
||||
def own_plan(self, forward_batch, *, runner_slots: bool = False) -> KVLocPlan:
|
||||
|
||||
@@ -22,7 +22,8 @@ forwards are in flight. So:
|
||||
|
||||
* The write window is translated once per sub-pool, by the plan: the
|
||||
full-attention ids when the plan is built, another sub-pool's on first
|
||||
use (`write_ids`). Every writer binds them.
|
||||
use (`write_ids`), or both by the producer's fused gather. Every writer
|
||||
binds them.
|
||||
* A read is translated exactly once, only by the translator's read
|
||||
primitives (`KVIndexTranslator`); a consumer never translates.
|
||||
* The read table (`read_table`) exists only once a table reader asks for
|
||||
@@ -130,6 +131,7 @@ class KVLocPlan:
|
||||
write_virtual: Optional[torch.Tensor],
|
||||
read_extent: int = 0,
|
||||
write_slots: Optional[torch.Tensor] = None,
|
||||
translated_writes: Optional[Dict[IdSpaceKind, torch.Tensor]] = None,
|
||||
):
|
||||
self._source = source
|
||||
self.req_pool_indices = req_pool_indices
|
||||
@@ -139,34 +141,25 @@ class KVLocPlan:
|
||||
self.seq_lens_cpu = seq_lens_cpu
|
||||
self.read_extent = read_extent
|
||||
full = source.space(IdSpaceKind.FULL)
|
||||
# Scheduler accept/reclaim bookkeeping still names these virtual ids.
|
||||
self.write_virtual = write_virtual
|
||||
self.write_physical = write_slots
|
||||
self._full_key = full.key
|
||||
self._write_ids: Dict[Hashable, Optional[torch.Tensor]] = {}
|
||||
if write_slots is not None:
|
||||
# A runner's own write buffer (graph capture, warmup): already in
|
||||
# its pool's ids, naming the sink until a replay fills it, and kept
|
||||
# by address by a captured graph.
|
||||
# A runner's capture-stable buffer is already in its pool's ids.
|
||||
# Without virtual ids, other sub-pools use sink slots at capture.
|
||||
assert write_virtual is None
|
||||
self.write_virtual = None if full.write is not None else write_slots
|
||||
self.write_physical = write_slots
|
||||
else:
|
||||
# Aliases the ScheduleBatch's tensor, which stays virtual for the
|
||||
# radix tree, the accept path and lazy compaction's in-flight write
|
||||
# set.
|
||||
self.write_virtual = write_virtual
|
||||
# FIXME: a speculative iteration builds its plan before its draft
|
||||
# runs, so this launch sits on the host path ahead of the draft's
|
||||
# graph, idle GPU time at low concurrency. Translate the window
|
||||
# inside the iteration's first captured forward instead, into a
|
||||
# capture-stable buffer the plan owns, which every later forward
|
||||
# and direct writer reads -- once per sub-pool, the sliding-window
|
||||
# ids (derived in `write_ids`) included.
|
||||
self.write_physical = (
|
||||
full.write(write_virtual)
|
||||
if full.write is not None and write_virtual is not None
|
||||
else write_virtual
|
||||
)
|
||||
self._full_key = full.key
|
||||
self._write_ids: Dict[Hashable, Optional[torch.Tensor]] = {
|
||||
full.key: self.write_physical
|
||||
}
|
||||
self._write_ids[full.key] = write_slots
|
||||
if translated_writes is not None:
|
||||
assert write_slots is None and write_virtual is not None
|
||||
for kind, ids in translated_writes.items():
|
||||
space = source.space(kind)
|
||||
assert space is not None and space.write is not None
|
||||
assert ids.shape == write_virtual.shape
|
||||
self._write_ids[space.key] = ids
|
||||
self.write_physical = self.write_ids(source)
|
||||
self._read_tables: Dict[Hashable, KVIndexTable] = {}
|
||||
|
||||
# -- writes ----------------------------------------------------------------
|
||||
|
||||
@@ -6,7 +6,6 @@ from typing import Optional, Union
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.speculative.cache_locs import assign_extend_cache_locs_func
|
||||
from sglang.kernels.ops.speculative.dspark.dspark_schedule import (
|
||||
ScheduleVerifyLensTopk,
|
||||
compute_sort_survival,
|
||||
@@ -796,27 +795,18 @@ def alloc_verify_window(
|
||||
prefix_lens = batch.seq_lens
|
||||
verify_w = verify_num_draft_tokens
|
||||
positions_2d = prefix_lens.unsqueeze(1) + block_pos_offsets
|
||||
verify_cache_loc = assign_extend_cache_locs_func(
|
||||
plan = model_runner.kv_index_translator.plan_uniform_extend(
|
||||
req_pool_indices=batch.req_pool_indices,
|
||||
req_to_token=model_runner.req_to_token_pool.req_to_token,
|
||||
start_offset=prefix_lens,
|
||||
end_offset=prefix_lens + verify_w,
|
||||
batch_size=bs,
|
||||
draft_token_num=verify_w,
|
||||
device=device,
|
||||
seq_lens=prefix_lens,
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
width=verify_w,
|
||||
)
|
||||
verify_cache_loc_2d = verify_cache_loc.view(bs, verify_w)
|
||||
verify_cache_loc = plan.write_virtual
|
||||
return VerifyWindow(
|
||||
positions_2d=positions_2d,
|
||||
verify_cache_loc=verify_cache_loc,
|
||||
verify_cache_loc_2d=verify_cache_loc_2d,
|
||||
kv_loc_plan=model_runner.kv_index_translator.plan(
|
||||
req_pool_indices=batch.req_pool_indices,
|
||||
seq_lens=prefix_lens,
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
write_virtual=verify_cache_loc,
|
||||
read_extent=verify_w,
|
||||
),
|
||||
verify_cache_loc_2d=verify_cache_loc.view(bs, verify_w),
|
||||
kv_loc_plan=plan,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user