From 2e9b2cacda6af490fc26eb918b7bc236fbc59fc3 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Sun, 18 Oct 2026 05:41:34 -0700 Subject: [PATCH] [Perf] speculative: Fuse draft write-window gathering and KV translation (#44972) --- .../kernels/ops/speculative/__init__.py | 1 + .../kernels/ops/speculative/cache_locs.py | 52 +++++++- .../srt/mem_cache/kv_index_translator.py | 65 +++++++++- python/sglang/srt/mem_cache/kv_loc_plan.py | 45 +++---- .../dspark_components/dspark_planner.py | 24 +--- .../test_uniform_extend_translation.py | 78 +++++++++++ .../unit/mem_cache/test_kv_loc_plan.py | 28 +++- .../mem_cache/test_kv_plan_uniform_extend.py | 121 ++++++++++++++++++ 8 files changed, 365 insertions(+), 49 deletions(-) create mode 100644 test/registered/kernels/ops/speculative/test_uniform_extend_translation.py create mode 100644 test/registered/unit/mem_cache/test_kv_plan_uniform_extend.py diff --git a/python/sglang/kernels/ops/speculative/__init__.py b/python/sglang/kernels/ops/speculative/__init__.py index 1dfa822ad8..9446e035aa 100644 --- a/python/sglang/kernels/ops/speculative/__init__.py +++ b/python/sglang/kernels/ops/speculative/__init__.py @@ -16,6 +16,7 @@ from sglang.kernels.spec import ( # (module, public_fn) migrated from speculative/triton_ops. _TRITON_KERNELS = [ ("cache_locs", "assign_extend_cache_locs_func"), + ("cache_locs", "assign_extend_cache_locs_uniform_func"), ("cache_locs", "generate_draft_decode_kv_indices"), ("cache_locs", "generate_draft_decode_window_kv_indices"), ("eagle", "fill_bonus_tokens"), diff --git a/python/sglang/kernels/ops/speculative/cache_locs.py b/python/sglang/kernels/ops/speculative/cache_locs.py index 9ed4ecbbee..a93975e392 100644 --- a/python/sglang/kernels/ops/speculative/cache_locs.py +++ b/python/sglang/kernels/ops/speculative/cache_locs.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections.abc import Callable +from typing import Optional import torch import triton @@ -619,6 +620,13 @@ def assign_extend_cache_locs_uniform( out_cache_loc, pool_len: tl.constexpr, draft_token_num: tl.constexpr, + full_v2p=None, + swa_v2p=None, + full_out=None, + swa_out=None, + PAGE_SIZE: tl.constexpr = 1, + num_v_pages=0, + num_swa_pages=0, ): """Uniform-length variant of assign_extend_cache_locs: every row extends exactly draft_token_num tokens, so the end offset is start + @@ -637,6 +645,21 @@ def assign_extend_cache_locs_uniform( mask = o < draft_token_num data = tl.load(token_pool + kv_start + o, mask=mask) tl.store(out_cache_ptr + o, data, mask=mask) + if full_v2p is not None: + page = data // PAGE_SIZE + valid = mask & (data >= 0) & (page < num_v_pages) + physical = tl.load( + full_v2p + tl.where(valid, page, 0), mask=valid, other=-1 + ) + full_loc = tl.maximum(physical * PAGE_SIZE + data % PAGE_SIZE, 0) + tl.store(full_out + pid * draft_token_num + o, full_loc, mask=mask) + if swa_v2p is not None: + swa_valid = mask & (data >= 0) & (page < num_swa_pages) + physical = tl.load( + swa_v2p + tl.where(swa_valid, page, 0), mask=swa_valid, other=-1 + ) + swa_loc = tl.maximum(physical * PAGE_SIZE + data % PAGE_SIZE, 0) + tl.store(swa_out + pid * draft_token_num + o, swa_loc, mask=mask) def assign_extend_cache_locs_uniform_func( @@ -646,11 +669,30 @@ def assign_extend_cache_locs_uniform_func( batch_size: int, draft_token_num: int, device, + *, + full_v2p: Optional[torch.Tensor] = None, + swa_v2p: Optional[torch.Tensor] = None, + full_out: Optional[torch.Tensor] = None, + swa_out: Optional[torch.Tensor] = None, + page_size: int = 1, ) -> torch.Tensor: """assign_extend_cache_locs for the uniform case (all rows extend exactly draft_token_num tokens, e.g. spec target-verify prep). Computes end offsets inside the kernel, removing the eager `seq_lens + draft_token_num` - add from the host critical path.""" + add from the host critical path. Optional v2p tables fill physical FULL + and SWA write windows in the same launch; the returned ids stay virtual. + Mapping tables and output buffers must be contiguous, with one output + per gathered token. Translation is rank-local (no DCP widening). + """ + if full_v2p is not None: + assert full_out is not None and full_out.numel() == batch_size * draft_token_num + assert full_v2p.is_contiguous() and full_out.is_contiguous() + assert swa_v2p is None or ( + swa_out is not None + and swa_out.numel() == full_out.numel() + and swa_out.is_contiguous() + and swa_v2p.is_contiguous() + ) if _is_cuda or _is_hip or _is_musa or _is_xpu: out_cache_loc = torch.empty( (batch_size * draft_token_num,), @@ -664,10 +706,18 @@ def assign_extend_cache_locs_uniform_func( out_cache_loc, req_to_token.shape[1], draft_token_num, + full_v2p=full_v2p, + swa_v2p=swa_v2p, + full_out=full_out, + swa_out=swa_out, + PAGE_SIZE=page_size, + num_v_pages=0 if full_v2p is None else full_v2p.numel(), + num_swa_pages=0 if swa_v2p is None else swa_v2p.numel(), ) return out_cache_loc # NPU / CPU platforms: fall back to the end_offset-tensor path. + assert full_v2p is None, "fused write translation requires a Triton device" return assign_extend_cache_locs_func( req_pool_indices=req_pool_indices, req_to_token=req_to_token, diff --git a/python/sglang/srt/mem_cache/kv_index_translator.py b/python/sglang/srt/mem_cache/kv_index_translator.py index e97fd75501..8d58e9d98c 100644 --- a/python/sglang/srt/mem_cache/kv_index_translator.py +++ b/python/sglang/srt/mem_cache/kv_index_translator.py @@ -25,8 +25,9 @@ The two coincide on a plain pool, so nothing here does any work there. A pool can hold several sub-pools (full attention, sliding window), each with its own physical ids and named by an `IdSpace` (`space`). Writes are translated -once per sub-pool, by the iteration's `KVLocPlan`. A read is translated exactly -once, by the readers below and nowhere else; a backend never translates. The +once per sub-pool, by the iteration's `KVLocPlan` or its fused producer. +A read is translated exactly once, by the readers below or a captured gather +using the `IdSpace`'s live mapping. The plan's read table exists only once a table reader asks for it (`read_table`, `copy_page_table`), built into that reader's own capture-stable table when it is the first; later readers take it, earlier ones translate in their own @@ -276,12 +277,15 @@ class KVIndexTranslator: write_virtual: Optional[torch.Tensor], read_extent: int = 0, write_slots: Optional[torch.Tensor] = None, + translated_writes: Optional[Dict[IdSpaceKind, torch.Tensor]] = None, ) -> KVLocPlan: """This iteration's ids, translated once. ``write_virtual`` is the iteration's write window (``[batch, window]`` flattened when its forwards write columns of it); ``read_extent`` is how far past ``seq_lens`` the iteration's reads reach. ``write_slots`` replaces - ``write_virtual`` for a runner's own buffer (`bind_runner_slots`).""" + ``write_virtual`` for a runner's own buffer (`bind_runner_slots`). + ``translated_writes`` supplies physical outputs already computed by + the producer, keyed by this source's sub-pools.""" return KVLocPlan( source=self, req_pool_indices=req_pool_indices, @@ -290,6 +294,61 @@ class KVIndexTranslator: write_virtual=write_virtual, read_extent=read_extent, write_slots=write_slots, + translated_writes=translated_writes, + ) + + def plan_uniform_extend( + self, + *, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_cpu: Optional[torch.Tensor], + width: int, + ) -> KVLocPlan: + """Gather one fixed-width speculative window, translating at its producer. + + The virtual window stays available for accept/reclaim bookkeeping; + draft, target and direct writers share the physical outputs. DCP and + non-CUDA devices retain the plan's ordinary write translation. + """ + from sglang.kernels.ops.speculative.cache_locs import ( + assign_extend_cache_locs_uniform_func, + ) + + translated = None + mappings = {} + if req_pool_indices.is_cuda and self.reads_are_translated: + translated = {} + for kind in (IdSpaceKind.FULL, IdSpaceKind.SLIDING_WINDOW): + space = self.space(kind) + if space is not None and space.read_v2p is not None: + mappings[kind] = space.read_v2p + translated[kind] = torch.empty( + req_pool_indices.numel() * width, + dtype=torch.int64, + device=req_pool_indices.device, + ) + writes = translated or {} + virtual = assign_extend_cache_locs_uniform_func( + req_pool_indices=req_pool_indices, + req_to_token=self.req_to_token, + start_offset=seq_lens, + batch_size=req_pool_indices.numel(), + draft_token_num=width, + device=req_pool_indices.device, + full_v2p=mappings.get(IdSpaceKind.FULL), + swa_v2p=mappings.get(IdSpaceKind.SLIDING_WINDOW), + full_out=writes.get(IdSpaceKind.FULL), + swa_out=writes.get(IdSpaceKind.SLIDING_WINDOW), + page_size=self.page_size, + ) + return self.plan( + req_pool_indices=req_pool_indices, + seq_lens=seq_lens, + seq_lens_cpu=seq_lens_cpu, + write_virtual=virtual, + read_extent=width, + translated_writes=translated, ) def own_plan(self, forward_batch, *, runner_slots: bool = False) -> KVLocPlan: diff --git a/python/sglang/srt/mem_cache/kv_loc_plan.py b/python/sglang/srt/mem_cache/kv_loc_plan.py index 1ecb9dc866..eea9619871 100644 --- a/python/sglang/srt/mem_cache/kv_loc_plan.py +++ b/python/sglang/srt/mem_cache/kv_loc_plan.py @@ -22,7 +22,8 @@ forwards are in flight. So: * The write window is translated once per sub-pool, by the plan: the full-attention ids when the plan is built, another sub-pool's on first - use (`write_ids`). Every writer binds them. + use (`write_ids`), or both by the producer's fused gather. Every writer + binds them. * A read is translated exactly once, only by the translator's read primitives (`KVIndexTranslator`); a consumer never translates. * The read table (`read_table`) exists only once a table reader asks for @@ -130,6 +131,7 @@ class KVLocPlan: write_virtual: Optional[torch.Tensor], read_extent: int = 0, write_slots: Optional[torch.Tensor] = None, + translated_writes: Optional[Dict[IdSpaceKind, torch.Tensor]] = None, ): self._source = source self.req_pool_indices = req_pool_indices @@ -139,34 +141,25 @@ class KVLocPlan: self.seq_lens_cpu = seq_lens_cpu self.read_extent = read_extent full = source.space(IdSpaceKind.FULL) + # Scheduler accept/reclaim bookkeeping still names these virtual ids. + self.write_virtual = write_virtual + self.write_physical = write_slots + self._full_key = full.key + self._write_ids: Dict[Hashable, Optional[torch.Tensor]] = {} if write_slots is not None: - # A runner's own write buffer (graph capture, warmup): already in - # its pool's ids, naming the sink until a replay fills it, and kept - # by address by a captured graph. + # A runner's capture-stable buffer is already in its pool's ids. + # Without virtual ids, other sub-pools use sink slots at capture. assert write_virtual is None self.write_virtual = None if full.write is not None else write_slots - self.write_physical = write_slots - else: - # Aliases the ScheduleBatch's tensor, which stays virtual for the - # radix tree, the accept path and lazy compaction's in-flight write - # set. - self.write_virtual = write_virtual - # FIXME: a speculative iteration builds its plan before its draft - # runs, so this launch sits on the host path ahead of the draft's - # graph, idle GPU time at low concurrency. Translate the window - # inside the iteration's first captured forward instead, into a - # capture-stable buffer the plan owns, which every later forward - # and direct writer reads -- once per sub-pool, the sliding-window - # ids (derived in `write_ids`) included. - self.write_physical = ( - full.write(write_virtual) - if full.write is not None and write_virtual is not None - else write_virtual - ) - self._full_key = full.key - self._write_ids: Dict[Hashable, Optional[torch.Tensor]] = { - full.key: self.write_physical - } + self._write_ids[full.key] = write_slots + if translated_writes is not None: + assert write_slots is None and write_virtual is not None + for kind, ids in translated_writes.items(): + space = source.space(kind) + assert space is not None and space.write is not None + assert ids.shape == write_virtual.shape + self._write_ids[space.key] = ids + self.write_physical = self.write_ids(source) self._read_tables: Dict[Hashable, KVIndexTable] = {} # -- writes ---------------------------------------------------------------- diff --git a/python/sglang/srt/speculative/dspark_components/dspark_planner.py b/python/sglang/srt/speculative/dspark_components/dspark_planner.py index f0793f3fc5..34198b4de5 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_planner.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_planner.py @@ -6,7 +6,6 @@ from typing import Optional, Union import msgspec import torch -from sglang.kernels.ops.speculative.cache_locs import assign_extend_cache_locs_func from sglang.kernels.ops.speculative.dspark.dspark_schedule import ( ScheduleVerifyLensTopk, compute_sort_survival, @@ -796,27 +795,18 @@ def alloc_verify_window( prefix_lens = batch.seq_lens verify_w = verify_num_draft_tokens positions_2d = prefix_lens.unsqueeze(1) + block_pos_offsets - verify_cache_loc = assign_extend_cache_locs_func( + plan = model_runner.kv_index_translator.plan_uniform_extend( req_pool_indices=batch.req_pool_indices, - req_to_token=model_runner.req_to_token_pool.req_to_token, - start_offset=prefix_lens, - end_offset=prefix_lens + verify_w, - batch_size=bs, - draft_token_num=verify_w, - device=device, + seq_lens=prefix_lens, + seq_lens_cpu=seq_lens_cpu, + width=verify_w, ) - verify_cache_loc_2d = verify_cache_loc.view(bs, verify_w) + verify_cache_loc = plan.write_virtual return VerifyWindow( positions_2d=positions_2d, verify_cache_loc=verify_cache_loc, - verify_cache_loc_2d=verify_cache_loc_2d, - kv_loc_plan=model_runner.kv_index_translator.plan( - req_pool_indices=batch.req_pool_indices, - seq_lens=prefix_lens, - seq_lens_cpu=seq_lens_cpu, - write_virtual=verify_cache_loc, - read_extent=verify_w, - ), + verify_cache_loc_2d=verify_cache_loc.view(bs, verify_w), + kv_loc_plan=plan, ) diff --git a/test/registered/kernels/ops/speculative/test_uniform_extend_translation.py b/test/registered/kernels/ops/speculative/test_uniform_extend_translation.py new file mode 100644 index 0000000000..f6abdc9cd6 --- /dev/null +++ b/test/registered/kernels/ops/speculative/test_uniform_extend_translation.py @@ -0,0 +1,78 @@ +"""Fused write gathering must reuse kernels across pool sizes and sink bad IDs.""" + +import unittest + +import torch + +from sglang.kernels.ops.speculative.cache_locs import ( + assign_extend_cache_locs_uniform, + assign_extend_cache_locs_uniform_func, +) +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="1-gpu-large") + + +class TestUniformExtendTranslation(CustomTestCase): + def test_pool_sizes_are_runtime_bounds(self): + """Different virtual table lengths must share the warmed kernel. + Negative, tombstoned and past-table slots map to the sink independently + in FULL and SWA, including when their virtual tables differ in size. + """ + ps = 128 + tokens = [ + -129, + -1, + 0, + 3 * ps + 7, + 62 * ps + 1, + 126 * ps + 2, + 254 * ps + 3, + 511 * ps, + ] + req_to_token = torch.tensor([tokens], device="cuda", dtype=torch.int64) + rows = torch.zeros(1, device="cuda", dtype=torch.int64) + starts = torch.zeros_like(rows) + full_out = torch.empty(8, device="cuda", dtype=torch.int64) + swa_out = torch.empty_like(full_out) + cache = assign_extend_cache_locs_uniform.device_caches[ + torch.cuda.current_device() + ][0] + saved = dict(cache) + cache.clear() + try: + for full_size, swa_size in ((127, 63), (255, 127), (511, 255), (127, 511)): + full = torch.arange(full_size, device="cuda", dtype=torch.int64) + swa = torch.arange(swa_size, device="cuda", dtype=torch.int64) + 1 + full[0] = swa[0] = 0 + full[3] = swa[3] = -1 + got_virtual = assign_extend_cache_locs_uniform_func( + rows, + req_to_token, + starts, + 1, + 8, + "cuda", + full_v2p=full, + swa_v2p=swa, + full_out=full_out, + swa_out=swa_out, + page_size=ps, + ) + torch.testing.assert_close(got_virtual, req_to_token.flatten()) + for mapping, output in ((full, full_out), (swa, swa_out)): + table = mapping.cpu().tolist() + expected = [] + for token in tokens: + page, offset = divmod(token, ps) + physical = table[page] if 0 <= page < len(table) else -1 + expected.append(max(physical * ps + offset, 0)) + torch.testing.assert_close(output, output.new_tensor(expected)) + self.assertEqual(len(cache), 1, (full_size, swa_size)) + finally: + cache.update(saved) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/mem_cache/test_kv_loc_plan.py b/test/registered/unit/mem_cache/test_kv_loc_plan.py index 6712eff160..341eb47f57 100644 --- a/test/registered/unit/mem_cache/test_kv_loc_plan.py +++ b/test/registered/unit/mem_cache/test_kv_loc_plan.py @@ -55,8 +55,9 @@ from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.runtime_context import publish, reset_context from sglang.srt.server_args import ServerArgs from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase -register_cpu_ci(est_time=8, suite="base-a-test-cpu") +register_cpu_ci(est_time=10, suite="base-a-test-cpu") _DEV = "cpu" _PS = 2 @@ -126,7 +127,7 @@ class _FakeUnifiedSWAKVPool: self._swa_allocator = swa_allocator -class TestKVLocPlan(unittest.TestCase): +class TestKVLocPlan(CustomTestCase): def setUp(self): # `KVIndexTranslator.__init__` reads `attn_dcp_size`, a derived # parallel width that only exists once a config is published. @@ -229,6 +230,29 @@ class TestKVLocPlan(unittest.TestCase): self.assertIs(plan.write_ids(self.private_draft), self.window) self.assertTrue(torch.equal(self.window, window)) # never mutated + def test_precomputed_writes_seed_each_space_independently(self): + expected = { + kind: self.target.space(kind).write(self.window) for kind in (_FULL, _SWA) + } + for kinds in ((_FULL,), (_SWA,), (_FULL, _SWA)): + with self.subTest(precomputed=kinds): + supplied = {kind: expected[kind] for kind in kinds} + plan = self.target.plan( + req_pool_indices=self.rpi, + seq_lens=self.seq_lens, + seq_lens_cpu=self.seq_lens.clone(), + write_virtual=self.window, + translated_writes=supplied, + ) + for kind in (_FULL, _SWA): + ids = plan.write_ids(self.target, kind=kind) + torch.testing.assert_close(ids, expected[kind]) + self.assertIs(plan.write_ids(self.target, kind=kind), ids) + if kind in supplied: + self.assertIs(ids, supplied[kind]) + self.assertIs(plan.write_ids(self.fused_draft), plan.write_physical) + self.assertIs(plan.write_ids(self.private_draft), self.window) + def test_cols_slice_names_a_split_batch_s_tokens(self): """Two-batch overlap splits a forward's tokens; each half writes its own tokens of the window, whatever the forward's own columns were.""" diff --git a/test/registered/unit/mem_cache/test_kv_plan_uniform_extend.py b/test/registered/unit/mem_cache/test_kv_plan_uniform_extend.py new file mode 100644 index 0000000000..6042f64148 --- /dev/null +++ b/test/registered/unit/mem_cache/test_kv_plan_uniform_extend.py @@ -0,0 +1,121 @@ +"""GPU producer/plan contract: virtual bookkeeping and shared physical writes.""" + +import unittest +from types import SimpleNamespace + +import torch + +from sglang.kernels.ops.memory.virtual_slot import translate_token_ids +from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator +from sglang.srt.mem_cache.kv_loc_plan import IdSpace, IdSpaceKind +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=15, stage="extra-a", runner_config="1-gpu-small") + + +@unittest.skipUnless(torch.cuda.is_available(), "requires CUDA") +class TestUniformExtendPlan(CustomTestCase): + def test_shared_writes_preserve_virtual_ids_and_page_offsets(self): + """A fused gather must retain per-page offsets and every reader's space. + + Changing v2p after plan creation must not retranslate that iteration's + writes; the next iteration must pick up the new map. This catches + losing the producer's outputs or incorrectly sharing FULL with SWA. + """ + for ps in (1, 128): + for width in (7, 8, 65): + for swa_enabled in (False, True): + with self.subTest(page_size=ps, width=width, swa=swa_enabled): + self._check_plan(ps, width, swa_enabled) + + def _check_plan(self, ps, width, swa_enabled): + full_kind, swa_kind = IdSpaceKind.FULL, IdSpaceKind.SLIDING_WINDOW + row_tokens = ((2 * ps + width + ps - 1) // ps) * ps + page_count = 3 * row_tokens // ps + req_to_token = torch.arange(3 * row_tokens, device="cuda").view(3, row_tokens) + req_to_token[0].zero_() + full = ( + torch.arange(page_count, device="cuda", dtype=torch.int64) + .flip(0) + .contiguous() + ) + full[0] = 0 + swa = torch.arange(page_count, device="cuda", dtype=torch.int64) + swa[row_tokens // ps + 1] = -1 + + def space(key, table): + return IdSpace( + key=key, + read_v2p=table, + write=lambda ids: translate_token_ids(loc=ids, v2p=table, page_size=ps), + ) + + source = KVIndexTranslator.__new__(KVIndexTranslator) + source.page_size = ps + source.req_to_token = req_to_token + source._spaces = {full_kind: space("full", full)} + if swa_enabled: + source._spaces[swa_kind] = space("swa", swa) + fused = KVIndexTranslator.__new__(KVIndexTranslator) + fused._spaces = {full_kind: source.space(full_kind)} + private = KVIndexTranslator.__new__(KVIndexTranslator) + private._spaces = {full_kind: IdSpace(key="private")} + rows = torch.tensor([2, 1, 0], device="cuda") + lengths = torch.tensor([ps - 1, ps + 1, 0], device="cuda") + expected_virtual = req_to_token[ + rows[:, None], lengths[:, None] + torch.arange(width, device="cuda") + ].flatten() + + for iteration in range(2): + if iteration: + full[1:] = 2 + swa[1:] = 3 + plan = source.plan_uniform_extend( + req_pool_indices=rows, + seq_lens=lengths, + seq_lens_cpu=lengths.cpu(), + width=width, + ) + expected = {} + for kind, mapping in ((full_kind, full), (swa_kind, swa)): + if source.space(kind) is not None: + expected[kind] = ( + mapping[expected_virtual // ps] * ps + expected_virtual % ps + ).clamp_min(0) + torch.testing.assert_close(plan.write_virtual, expected_virtual) + # Retain this iteration's ids even if a later allocator update + # changes the table before another consumer asks for its view. + full[1:] = 4 + swa[1:] = 5 + torch.testing.assert_close(plan.write_ids(source), expected[full_kind]) + self.assertIs(plan.write_ids(fused), plan.write_ids(source)) + torch.testing.assert_close(plan.write_ids(private), expected_virtual) + if swa_enabled: + torch.testing.assert_close( + plan.write_ids(source, kind=swa_kind), expected[swa_kind] + ) + fb = SimpleNamespace() + plan.bind(fb, fused, cols=slice(0, width - 1)) + torch.testing.assert_close( + fb.out_cache_loc, expected[full_kind].view(3, width)[:, :-1].reshape(-1) + ) + torch.testing.assert_close( + fb.out_cache_loc_virtual, + expected_virtual.view(3, width)[:, :-1].reshape(-1), + ) + packed = torch.tensor([width + 1, -1, 0], device="cuda") + torch.testing.assert_close( + plan.write_ids(source, cols=packed), + torch.stack( + ( + expected[full_kind][width + 1], + expected[full_kind].new_zeros(()), + expected[full_kind][0], + ) + ), + ) + + +if __name__ == "__main__": + unittest.main()