[Perf] speculative: Fuse draft write-window gathering and KV translation (#44972)
This commit is contained in:
@@ -16,6 +16,7 @@ from sglang.kernels.spec import (
|
||||
# (module, public_fn) migrated from speculative/triton_ops.
|
||||
_TRITON_KERNELS = [
|
||||
("cache_locs", "assign_extend_cache_locs_func"),
|
||||
("cache_locs", "assign_extend_cache_locs_uniform_func"),
|
||||
("cache_locs", "generate_draft_decode_kv_indices"),
|
||||
("cache_locs", "generate_draft_decode_window_kv_indices"),
|
||||
("eagle", "fill_bonus_tokens"),
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
import triton
|
||||
@@ -619,6 +620,13 @@ def assign_extend_cache_locs_uniform(
|
||||
out_cache_loc,
|
||||
pool_len: tl.constexpr,
|
||||
draft_token_num: tl.constexpr,
|
||||
full_v2p=None,
|
||||
swa_v2p=None,
|
||||
full_out=None,
|
||||
swa_out=None,
|
||||
PAGE_SIZE: tl.constexpr = 1,
|
||||
num_v_pages=0,
|
||||
num_swa_pages=0,
|
||||
):
|
||||
"""Uniform-length variant of assign_extend_cache_locs: every row extends
|
||||
exactly draft_token_num tokens, so the end offset is start +
|
||||
@@ -637,6 +645,21 @@ def assign_extend_cache_locs_uniform(
|
||||
mask = o < draft_token_num
|
||||
data = tl.load(token_pool + kv_start + o, mask=mask)
|
||||
tl.store(out_cache_ptr + o, data, mask=mask)
|
||||
if full_v2p is not None:
|
||||
page = data // PAGE_SIZE
|
||||
valid = mask & (data >= 0) & (page < num_v_pages)
|
||||
physical = tl.load(
|
||||
full_v2p + tl.where(valid, page, 0), mask=valid, other=-1
|
||||
)
|
||||
full_loc = tl.maximum(physical * PAGE_SIZE + data % PAGE_SIZE, 0)
|
||||
tl.store(full_out + pid * draft_token_num + o, full_loc, mask=mask)
|
||||
if swa_v2p is not None:
|
||||
swa_valid = mask & (data >= 0) & (page < num_swa_pages)
|
||||
physical = tl.load(
|
||||
swa_v2p + tl.where(swa_valid, page, 0), mask=swa_valid, other=-1
|
||||
)
|
||||
swa_loc = tl.maximum(physical * PAGE_SIZE + data % PAGE_SIZE, 0)
|
||||
tl.store(swa_out + pid * draft_token_num + o, swa_loc, mask=mask)
|
||||
|
||||
|
||||
def assign_extend_cache_locs_uniform_func(
|
||||
@@ -646,11 +669,30 @@ def assign_extend_cache_locs_uniform_func(
|
||||
batch_size: int,
|
||||
draft_token_num: int,
|
||||
device,
|
||||
*,
|
||||
full_v2p: Optional[torch.Tensor] = None,
|
||||
swa_v2p: Optional[torch.Tensor] = None,
|
||||
full_out: Optional[torch.Tensor] = None,
|
||||
swa_out: Optional[torch.Tensor] = None,
|
||||
page_size: int = 1,
|
||||
) -> torch.Tensor:
|
||||
"""assign_extend_cache_locs for the uniform case (all rows extend exactly
|
||||
draft_token_num tokens, e.g. spec target-verify prep). Computes end
|
||||
offsets inside the kernel, removing the eager `seq_lens + draft_token_num`
|
||||
add from the host critical path."""
|
||||
add from the host critical path. Optional v2p tables fill physical FULL
|
||||
and SWA write windows in the same launch; the returned ids stay virtual.
|
||||
Mapping tables and output buffers must be contiguous, with one output
|
||||
per gathered token. Translation is rank-local (no DCP widening).
|
||||
"""
|
||||
if full_v2p is not None:
|
||||
assert full_out is not None and full_out.numel() == batch_size * draft_token_num
|
||||
assert full_v2p.is_contiguous() and full_out.is_contiguous()
|
||||
assert swa_v2p is None or (
|
||||
swa_out is not None
|
||||
and swa_out.numel() == full_out.numel()
|
||||
and swa_out.is_contiguous()
|
||||
and swa_v2p.is_contiguous()
|
||||
)
|
||||
if _is_cuda or _is_hip or _is_musa or _is_xpu:
|
||||
out_cache_loc = torch.empty(
|
||||
(batch_size * draft_token_num,),
|
||||
@@ -664,10 +706,18 @@ def assign_extend_cache_locs_uniform_func(
|
||||
out_cache_loc,
|
||||
req_to_token.shape[1],
|
||||
draft_token_num,
|
||||
full_v2p=full_v2p,
|
||||
swa_v2p=swa_v2p,
|
||||
full_out=full_out,
|
||||
swa_out=swa_out,
|
||||
PAGE_SIZE=page_size,
|
||||
num_v_pages=0 if full_v2p is None else full_v2p.numel(),
|
||||
num_swa_pages=0 if swa_v2p is None else swa_v2p.numel(),
|
||||
)
|
||||
return out_cache_loc
|
||||
|
||||
# NPU / CPU platforms: fall back to the end_offset-tensor path.
|
||||
assert full_v2p is None, "fused write translation requires a Triton device"
|
||||
return assign_extend_cache_locs_func(
|
||||
req_pool_indices=req_pool_indices,
|
||||
req_to_token=req_to_token,
|
||||
|
||||
@@ -25,8 +25,9 @@ The two coincide on a plain pool, so nothing here does any work there.
|
||||
|
||||
A pool can hold several sub-pools (full attention, sliding window), each with
|
||||
its own physical ids and named by an `IdSpace` (`space`). Writes are translated
|
||||
once per sub-pool, by the iteration's `KVLocPlan`. A read is translated exactly
|
||||
once, by the readers below and nowhere else; a backend never translates. The
|
||||
once per sub-pool, by the iteration's `KVLocPlan` or its fused producer.
|
||||
A read is translated exactly once, by the readers below or a captured gather
|
||||
using the `IdSpace`'s live mapping. The
|
||||
plan's read table exists only once a table reader asks for it (`read_table`,
|
||||
`copy_page_table`), built into that reader's own capture-stable table when it
|
||||
is the first; later readers take it, earlier ones translate in their own
|
||||
@@ -276,12 +277,15 @@ class KVIndexTranslator:
|
||||
write_virtual: Optional[torch.Tensor],
|
||||
read_extent: int = 0,
|
||||
write_slots: Optional[torch.Tensor] = None,
|
||||
translated_writes: Optional[Dict[IdSpaceKind, torch.Tensor]] = None,
|
||||
) -> KVLocPlan:
|
||||
"""This iteration's ids, translated once. ``write_virtual`` is the
|
||||
iteration's write window (``[batch, window]`` flattened when its
|
||||
forwards write columns of it); ``read_extent`` is how far past
|
||||
``seq_lens`` the iteration's reads reach. ``write_slots`` replaces
|
||||
``write_virtual`` for a runner's own buffer (`bind_runner_slots`)."""
|
||||
``write_virtual`` for a runner's own buffer (`bind_runner_slots`).
|
||||
``translated_writes`` supplies physical outputs already computed by
|
||||
the producer, keyed by this source's sub-pools."""
|
||||
return KVLocPlan(
|
||||
source=self,
|
||||
req_pool_indices=req_pool_indices,
|
||||
@@ -290,6 +294,61 @@ class KVIndexTranslator:
|
||||
write_virtual=write_virtual,
|
||||
read_extent=read_extent,
|
||||
write_slots=write_slots,
|
||||
translated_writes=translated_writes,
|
||||
)
|
||||
|
||||
def plan_uniform_extend(
|
||||
self,
|
||||
*,
|
||||
req_pool_indices: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
seq_lens_cpu: Optional[torch.Tensor],
|
||||
width: int,
|
||||
) -> KVLocPlan:
|
||||
"""Gather one fixed-width speculative window, translating at its producer.
|
||||
|
||||
The virtual window stays available for accept/reclaim bookkeeping;
|
||||
draft, target and direct writers share the physical outputs. DCP and
|
||||
non-CUDA devices retain the plan's ordinary write translation.
|
||||
"""
|
||||
from sglang.kernels.ops.speculative.cache_locs import (
|
||||
assign_extend_cache_locs_uniform_func,
|
||||
)
|
||||
|
||||
translated = None
|
||||
mappings = {}
|
||||
if req_pool_indices.is_cuda and self.reads_are_translated:
|
||||
translated = {}
|
||||
for kind in (IdSpaceKind.FULL, IdSpaceKind.SLIDING_WINDOW):
|
||||
space = self.space(kind)
|
||||
if space is not None and space.read_v2p is not None:
|
||||
mappings[kind] = space.read_v2p
|
||||
translated[kind] = torch.empty(
|
||||
req_pool_indices.numel() * width,
|
||||
dtype=torch.int64,
|
||||
device=req_pool_indices.device,
|
||||
)
|
||||
writes = translated or {}
|
||||
virtual = assign_extend_cache_locs_uniform_func(
|
||||
req_pool_indices=req_pool_indices,
|
||||
req_to_token=self.req_to_token,
|
||||
start_offset=seq_lens,
|
||||
batch_size=req_pool_indices.numel(),
|
||||
draft_token_num=width,
|
||||
device=req_pool_indices.device,
|
||||
full_v2p=mappings.get(IdSpaceKind.FULL),
|
||||
swa_v2p=mappings.get(IdSpaceKind.SLIDING_WINDOW),
|
||||
full_out=writes.get(IdSpaceKind.FULL),
|
||||
swa_out=writes.get(IdSpaceKind.SLIDING_WINDOW),
|
||||
page_size=self.page_size,
|
||||
)
|
||||
return self.plan(
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
write_virtual=virtual,
|
||||
read_extent=width,
|
||||
translated_writes=translated,
|
||||
)
|
||||
|
||||
def own_plan(self, forward_batch, *, runner_slots: bool = False) -> KVLocPlan:
|
||||
|
||||
@@ -22,7 +22,8 @@ forwards are in flight. So:
|
||||
|
||||
* The write window is translated once per sub-pool, by the plan: the
|
||||
full-attention ids when the plan is built, another sub-pool's on first
|
||||
use (`write_ids`). Every writer binds them.
|
||||
use (`write_ids`), or both by the producer's fused gather. Every writer
|
||||
binds them.
|
||||
* A read is translated exactly once, only by the translator's read
|
||||
primitives (`KVIndexTranslator`); a consumer never translates.
|
||||
* The read table (`read_table`) exists only once a table reader asks for
|
||||
@@ -130,6 +131,7 @@ class KVLocPlan:
|
||||
write_virtual: Optional[torch.Tensor],
|
||||
read_extent: int = 0,
|
||||
write_slots: Optional[torch.Tensor] = None,
|
||||
translated_writes: Optional[Dict[IdSpaceKind, torch.Tensor]] = None,
|
||||
):
|
||||
self._source = source
|
||||
self.req_pool_indices = req_pool_indices
|
||||
@@ -139,34 +141,25 @@ class KVLocPlan:
|
||||
self.seq_lens_cpu = seq_lens_cpu
|
||||
self.read_extent = read_extent
|
||||
full = source.space(IdSpaceKind.FULL)
|
||||
# Scheduler accept/reclaim bookkeeping still names these virtual ids.
|
||||
self.write_virtual = write_virtual
|
||||
self.write_physical = write_slots
|
||||
self._full_key = full.key
|
||||
self._write_ids: Dict[Hashable, Optional[torch.Tensor]] = {}
|
||||
if write_slots is not None:
|
||||
# A runner's own write buffer (graph capture, warmup): already in
|
||||
# its pool's ids, naming the sink until a replay fills it, and kept
|
||||
# by address by a captured graph.
|
||||
# A runner's capture-stable buffer is already in its pool's ids.
|
||||
# Without virtual ids, other sub-pools use sink slots at capture.
|
||||
assert write_virtual is None
|
||||
self.write_virtual = None if full.write is not None else write_slots
|
||||
self.write_physical = write_slots
|
||||
else:
|
||||
# Aliases the ScheduleBatch's tensor, which stays virtual for the
|
||||
# radix tree, the accept path and lazy compaction's in-flight write
|
||||
# set.
|
||||
self.write_virtual = write_virtual
|
||||
# FIXME: a speculative iteration builds its plan before its draft
|
||||
# runs, so this launch sits on the host path ahead of the draft's
|
||||
# graph, idle GPU time at low concurrency. Translate the window
|
||||
# inside the iteration's first captured forward instead, into a
|
||||
# capture-stable buffer the plan owns, which every later forward
|
||||
# and direct writer reads -- once per sub-pool, the sliding-window
|
||||
# ids (derived in `write_ids`) included.
|
||||
self.write_physical = (
|
||||
full.write(write_virtual)
|
||||
if full.write is not None and write_virtual is not None
|
||||
else write_virtual
|
||||
)
|
||||
self._full_key = full.key
|
||||
self._write_ids: Dict[Hashable, Optional[torch.Tensor]] = {
|
||||
full.key: self.write_physical
|
||||
}
|
||||
self._write_ids[full.key] = write_slots
|
||||
if translated_writes is not None:
|
||||
assert write_slots is None and write_virtual is not None
|
||||
for kind, ids in translated_writes.items():
|
||||
space = source.space(kind)
|
||||
assert space is not None and space.write is not None
|
||||
assert ids.shape == write_virtual.shape
|
||||
self._write_ids[space.key] = ids
|
||||
self.write_physical = self.write_ids(source)
|
||||
self._read_tables: Dict[Hashable, KVIndexTable] = {}
|
||||
|
||||
# -- writes ----------------------------------------------------------------
|
||||
|
||||
@@ -6,7 +6,6 @@ from typing import Optional, Union
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.speculative.cache_locs import assign_extend_cache_locs_func
|
||||
from sglang.kernels.ops.speculative.dspark.dspark_schedule import (
|
||||
ScheduleVerifyLensTopk,
|
||||
compute_sort_survival,
|
||||
@@ -796,27 +795,18 @@ def alloc_verify_window(
|
||||
prefix_lens = batch.seq_lens
|
||||
verify_w = verify_num_draft_tokens
|
||||
positions_2d = prefix_lens.unsqueeze(1) + block_pos_offsets
|
||||
verify_cache_loc = assign_extend_cache_locs_func(
|
||||
plan = model_runner.kv_index_translator.plan_uniform_extend(
|
||||
req_pool_indices=batch.req_pool_indices,
|
||||
req_to_token=model_runner.req_to_token_pool.req_to_token,
|
||||
start_offset=prefix_lens,
|
||||
end_offset=prefix_lens + verify_w,
|
||||
batch_size=bs,
|
||||
draft_token_num=verify_w,
|
||||
device=device,
|
||||
seq_lens=prefix_lens,
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
width=verify_w,
|
||||
)
|
||||
verify_cache_loc_2d = verify_cache_loc.view(bs, verify_w)
|
||||
verify_cache_loc = plan.write_virtual
|
||||
return VerifyWindow(
|
||||
positions_2d=positions_2d,
|
||||
verify_cache_loc=verify_cache_loc,
|
||||
verify_cache_loc_2d=verify_cache_loc_2d,
|
||||
kv_loc_plan=model_runner.kv_index_translator.plan(
|
||||
req_pool_indices=batch.req_pool_indices,
|
||||
seq_lens=prefix_lens,
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
write_virtual=verify_cache_loc,
|
||||
read_extent=verify_w,
|
||||
),
|
||||
verify_cache_loc_2d=verify_cache_loc.view(bs, verify_w),
|
||||
kv_loc_plan=plan,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Fused write gathering must reuse kernels across pool sizes and sink bad IDs."""
|
||||
|
||||
import unittest
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.speculative.cache_locs import (
|
||||
assign_extend_cache_locs_uniform,
|
||||
assign_extend_cache_locs_uniform_func,
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="1-gpu-large")
|
||||
|
||||
|
||||
class TestUniformExtendTranslation(CustomTestCase):
|
||||
def test_pool_sizes_are_runtime_bounds(self):
|
||||
"""Different virtual table lengths must share the warmed kernel.
|
||||
Negative, tombstoned and past-table slots map to the sink independently
|
||||
in FULL and SWA, including when their virtual tables differ in size.
|
||||
"""
|
||||
ps = 128
|
||||
tokens = [
|
||||
-129,
|
||||
-1,
|
||||
0,
|
||||
3 * ps + 7,
|
||||
62 * ps + 1,
|
||||
126 * ps + 2,
|
||||
254 * ps + 3,
|
||||
511 * ps,
|
||||
]
|
||||
req_to_token = torch.tensor([tokens], device="cuda", dtype=torch.int64)
|
||||
rows = torch.zeros(1, device="cuda", dtype=torch.int64)
|
||||
starts = torch.zeros_like(rows)
|
||||
full_out = torch.empty(8, device="cuda", dtype=torch.int64)
|
||||
swa_out = torch.empty_like(full_out)
|
||||
cache = assign_extend_cache_locs_uniform.device_caches[
|
||||
torch.cuda.current_device()
|
||||
][0]
|
||||
saved = dict(cache)
|
||||
cache.clear()
|
||||
try:
|
||||
for full_size, swa_size in ((127, 63), (255, 127), (511, 255), (127, 511)):
|
||||
full = torch.arange(full_size, device="cuda", dtype=torch.int64)
|
||||
swa = torch.arange(swa_size, device="cuda", dtype=torch.int64) + 1
|
||||
full[0] = swa[0] = 0
|
||||
full[3] = swa[3] = -1
|
||||
got_virtual = assign_extend_cache_locs_uniform_func(
|
||||
rows,
|
||||
req_to_token,
|
||||
starts,
|
||||
1,
|
||||
8,
|
||||
"cuda",
|
||||
full_v2p=full,
|
||||
swa_v2p=swa,
|
||||
full_out=full_out,
|
||||
swa_out=swa_out,
|
||||
page_size=ps,
|
||||
)
|
||||
torch.testing.assert_close(got_virtual, req_to_token.flatten())
|
||||
for mapping, output in ((full, full_out), (swa, swa_out)):
|
||||
table = mapping.cpu().tolist()
|
||||
expected = []
|
||||
for token in tokens:
|
||||
page, offset = divmod(token, ps)
|
||||
physical = table[page] if 0 <= page < len(table) else -1
|
||||
expected.append(max(physical * ps + offset, 0))
|
||||
torch.testing.assert_close(output, output.new_tensor(expected))
|
||||
self.assertEqual(len(cache), 1, (full_size, swa_size))
|
||||
finally:
|
||||
cache.update(saved)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -55,8 +55,9 @@ from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.runtime_context import publish, reset_context
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=8, suite="base-a-test-cpu")
|
||||
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
|
||||
|
||||
_DEV = "cpu"
|
||||
_PS = 2
|
||||
@@ -126,7 +127,7 @@ class _FakeUnifiedSWAKVPool:
|
||||
self._swa_allocator = swa_allocator
|
||||
|
||||
|
||||
class TestKVLocPlan(unittest.TestCase):
|
||||
class TestKVLocPlan(CustomTestCase):
|
||||
def setUp(self):
|
||||
# `KVIndexTranslator.__init__` reads `attn_dcp_size`, a derived
|
||||
# parallel width that only exists once a config is published.
|
||||
@@ -229,6 +230,29 @@ class TestKVLocPlan(unittest.TestCase):
|
||||
self.assertIs(plan.write_ids(self.private_draft), self.window)
|
||||
self.assertTrue(torch.equal(self.window, window)) # never mutated
|
||||
|
||||
def test_precomputed_writes_seed_each_space_independently(self):
|
||||
expected = {
|
||||
kind: self.target.space(kind).write(self.window) for kind in (_FULL, _SWA)
|
||||
}
|
||||
for kinds in ((_FULL,), (_SWA,), (_FULL, _SWA)):
|
||||
with self.subTest(precomputed=kinds):
|
||||
supplied = {kind: expected[kind] for kind in kinds}
|
||||
plan = self.target.plan(
|
||||
req_pool_indices=self.rpi,
|
||||
seq_lens=self.seq_lens,
|
||||
seq_lens_cpu=self.seq_lens.clone(),
|
||||
write_virtual=self.window,
|
||||
translated_writes=supplied,
|
||||
)
|
||||
for kind in (_FULL, _SWA):
|
||||
ids = plan.write_ids(self.target, kind=kind)
|
||||
torch.testing.assert_close(ids, expected[kind])
|
||||
self.assertIs(plan.write_ids(self.target, kind=kind), ids)
|
||||
if kind in supplied:
|
||||
self.assertIs(ids, supplied[kind])
|
||||
self.assertIs(plan.write_ids(self.fused_draft), plan.write_physical)
|
||||
self.assertIs(plan.write_ids(self.private_draft), self.window)
|
||||
|
||||
def test_cols_slice_names_a_split_batch_s_tokens(self):
|
||||
"""Two-batch overlap splits a forward's tokens; each half writes its
|
||||
own tokens of the window, whatever the forward's own columns were."""
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
"""GPU producer/plan contract: virtual bookkeeping and shared physical writes."""
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.memory.virtual_slot import translate_token_ids
|
||||
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
|
||||
from sglang.srt.mem_cache.kv_loc_plan import IdSpace, IdSpaceKind
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(est_time=15, stage="extra-a", runner_config="1-gpu-small")
|
||||
|
||||
|
||||
@unittest.skipUnless(torch.cuda.is_available(), "requires CUDA")
|
||||
class TestUniformExtendPlan(CustomTestCase):
|
||||
def test_shared_writes_preserve_virtual_ids_and_page_offsets(self):
|
||||
"""A fused gather must retain per-page offsets and every reader's space.
|
||||
|
||||
Changing v2p after plan creation must not retranslate that iteration's
|
||||
writes; the next iteration must pick up the new map. This catches
|
||||
losing the producer's outputs or incorrectly sharing FULL with SWA.
|
||||
"""
|
||||
for ps in (1, 128):
|
||||
for width in (7, 8, 65):
|
||||
for swa_enabled in (False, True):
|
||||
with self.subTest(page_size=ps, width=width, swa=swa_enabled):
|
||||
self._check_plan(ps, width, swa_enabled)
|
||||
|
||||
def _check_plan(self, ps, width, swa_enabled):
|
||||
full_kind, swa_kind = IdSpaceKind.FULL, IdSpaceKind.SLIDING_WINDOW
|
||||
row_tokens = ((2 * ps + width + ps - 1) // ps) * ps
|
||||
page_count = 3 * row_tokens // ps
|
||||
req_to_token = torch.arange(3 * row_tokens, device="cuda").view(3, row_tokens)
|
||||
req_to_token[0].zero_()
|
||||
full = (
|
||||
torch.arange(page_count, device="cuda", dtype=torch.int64)
|
||||
.flip(0)
|
||||
.contiguous()
|
||||
)
|
||||
full[0] = 0
|
||||
swa = torch.arange(page_count, device="cuda", dtype=torch.int64)
|
||||
swa[row_tokens // ps + 1] = -1
|
||||
|
||||
def space(key, table):
|
||||
return IdSpace(
|
||||
key=key,
|
||||
read_v2p=table,
|
||||
write=lambda ids: translate_token_ids(loc=ids, v2p=table, page_size=ps),
|
||||
)
|
||||
|
||||
source = KVIndexTranslator.__new__(KVIndexTranslator)
|
||||
source.page_size = ps
|
||||
source.req_to_token = req_to_token
|
||||
source._spaces = {full_kind: space("full", full)}
|
||||
if swa_enabled:
|
||||
source._spaces[swa_kind] = space("swa", swa)
|
||||
fused = KVIndexTranslator.__new__(KVIndexTranslator)
|
||||
fused._spaces = {full_kind: source.space(full_kind)}
|
||||
private = KVIndexTranslator.__new__(KVIndexTranslator)
|
||||
private._spaces = {full_kind: IdSpace(key="private")}
|
||||
rows = torch.tensor([2, 1, 0], device="cuda")
|
||||
lengths = torch.tensor([ps - 1, ps + 1, 0], device="cuda")
|
||||
expected_virtual = req_to_token[
|
||||
rows[:, None], lengths[:, None] + torch.arange(width, device="cuda")
|
||||
].flatten()
|
||||
|
||||
for iteration in range(2):
|
||||
if iteration:
|
||||
full[1:] = 2
|
||||
swa[1:] = 3
|
||||
plan = source.plan_uniform_extend(
|
||||
req_pool_indices=rows,
|
||||
seq_lens=lengths,
|
||||
seq_lens_cpu=lengths.cpu(),
|
||||
width=width,
|
||||
)
|
||||
expected = {}
|
||||
for kind, mapping in ((full_kind, full), (swa_kind, swa)):
|
||||
if source.space(kind) is not None:
|
||||
expected[kind] = (
|
||||
mapping[expected_virtual // ps] * ps + expected_virtual % ps
|
||||
).clamp_min(0)
|
||||
torch.testing.assert_close(plan.write_virtual, expected_virtual)
|
||||
# Retain this iteration's ids even if a later allocator update
|
||||
# changes the table before another consumer asks for its view.
|
||||
full[1:] = 4
|
||||
swa[1:] = 5
|
||||
torch.testing.assert_close(plan.write_ids(source), expected[full_kind])
|
||||
self.assertIs(plan.write_ids(fused), plan.write_ids(source))
|
||||
torch.testing.assert_close(plan.write_ids(private), expected_virtual)
|
||||
if swa_enabled:
|
||||
torch.testing.assert_close(
|
||||
plan.write_ids(source, kind=swa_kind), expected[swa_kind]
|
||||
)
|
||||
fb = SimpleNamespace()
|
||||
plan.bind(fb, fused, cols=slice(0, width - 1))
|
||||
torch.testing.assert_close(
|
||||
fb.out_cache_loc, expected[full_kind].view(3, width)[:, :-1].reshape(-1)
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
fb.out_cache_loc_virtual,
|
||||
expected_virtual.view(3, width)[:, :-1].reshape(-1),
|
||||
)
|
||||
packed = torch.tensor([width + 1, -1, 0], device="cuda")
|
||||
torch.testing.assert_close(
|
||||
plan.write_ids(source, cols=packed),
|
||||
torch.stack(
|
||||
(
|
||||
expected[full_kind][width + 1],
|
||||
expected[full_kind].new_zeros(()),
|
||||
expected[full_kind][0],
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user