From b660100c008bc15f6e6979ddbaaed5a1c037350b Mon Sep 17 00:00:00 2001 From: Qiaolin Yu Date: Tue, 29 Sep 2026 21:52:28 -0700 Subject: [PATCH] [qwen 3.8 next] Fuse QSA graph replay metadata across draft steps (#41173) --- .../layers/attention/qsa/graph_metadata.py | 262 ++++++++++++------ .../attention/qwen_sparse_attn_backend.py | 71 +++++ .../kernels/ops/attention/qsa/test_qsa.py | 112 +++++++- 3 files changed, 349 insertions(+), 96 deletions(-) diff --git a/python/sglang/srt/layers/attention/qsa/graph_metadata.py b/python/sglang/srt/layers/attention/qsa/graph_metadata.py index dc91f87e0f..3ac793f961 100644 --- a/python/sglang/srt/layers/attention/qsa/graph_metadata.py +++ b/python/sglang/srt/layers/attention/qsa/graph_metadata.py @@ -5,9 +5,10 @@ a group's compressed slot is any of its raw slots floor-divided by the compress so per-row graph buffers are rebuilt from request lengths and ``req_to_token`` alone; accept-dependent speculative lengths never need the host. -Both kernels run once eagerly at capture warmup, -then are recorded into the main CUDA graph through ``init_forward_metadata_in_graph``; -all inputs are stable-address runner buffers. +Decode and target verify build row layout and compressed metadata in one kernel. +Draft extend first builds its variable-length row layout, then derives metadata +from those rows. Multi-step draft builds metadata for all steps in one launch. +The launchers refresh graph buffers on device before replay. """ from __future__ import annotations @@ -30,36 +31,15 @@ def _qsa_graph_layout_kernel( bs, num_tokens, num_padding, - extend_len, # uniform extend length (target verify); 0 -> extend_lens_ptr - MODE: tl.constexpr, # 0 = decode, 1 = target verify, 2 = draft extend ): pid = tl.program_id(0) - if MODE == 0: - if pid < bs: - real = pid < bs - num_padding - seq_len = tl.load(seq_lens_ptr + pid).to(tl.int32) - req = tl.load(req_pool_ptr + pid).to(tl.int64) - # Padding rows alias request slot 0: it is never allocated, so - # its pending-ring rows are the inert dump for their state - # stores, and its req_to_token row reads stay in-bounds. - req = tl.where(real, req, 0) - seq_len = tl.where(real, seq_len, 1) - prefix = tl.maximum(seq_len - 1, 0) - tl.store(row_seq_lens_ptr + pid, seq_len) - tl.store(row_req_pool_ptr + pid, req.to(tl.int32)) - tl.store(row_prefix_lens_ptr + pid, prefix) - return - real_reqs = bs - num_padding if pid == bs: # Tail program: dummy rows for the static capacity past the real layout. - if MODE == 1: - row_start = real_reqs * extend_len - else: - row_start = 0 - for j in range(real_reqs): - row_start += tl.load(extend_lens_ptr + j) + row_start = 0 + for j in range(real_reqs): + row_start += tl.load(extend_lens_ptr + j) for row in range(row_start, num_tokens): tl.store(row_seq_lens_ptr + row, 1) tl.store(row_prefix_lens_ptr + row, 0) @@ -68,24 +48,16 @@ def _qsa_graph_layout_kernel( tl.store(row_req_pool_ptr + row, 0) return - if MODE == 1: - eff = tl.where(pid < real_reqs, extend_len, 0) - offset = tl.minimum(pid, real_reqs) * extend_len - else: - eff = 0 - offset = 0 - for j in range(bs): - e_j = tl.where(j < real_reqs, tl.load(extend_lens_ptr + j), 0) - offset += tl.where(j < pid, e_j, 0) - eff = tl.where(j == pid, e_j, eff) + eff = 0 + offset = 0 + for j in range(bs): + e_j = tl.where(j < real_reqs, tl.load(extend_lens_ptr + j), 0) + offset += tl.where(j < pid, e_j, 0) + eff = tl.where(j == pid, e_j, eff) base = tl.load(seq_lens_ptr + pid).to(tl.int32) req = tl.load(req_pool_ptr + pid).to(tl.int64) - if MODE == 1: - prefix = base - limit = base + eff - else: - prefix = tl.maximum(base - eff, 0) - limit = base + prefix = tl.maximum(base - eff, 0) + limit = base for j in range(eff): row = offset + j seq_len = tl.minimum(prefix + 1 + j, limit) @@ -113,30 +85,55 @@ def _qsa_graph_row_metadata_kernel( RATIO: tl.constexpr, FULL_PAGE: tl.constexpr, # full-KV tokens per page PAGE_BLOCK: tl.constexpr, + seq_lens_ptr, + req_pool_ptr, + row_prefix_lens_ptr, + bs, + num_padding, + extend_len, + seq_offset, + MODE: tl.constexpr, ): row = tl.program_id(0) - seq_len = tl.load(row_seq_lens_ptr + row).to(tl.int32) - req = tl.load(row_req_pool_ptr + row).to(tl.int64) + if MODE == 0 or MODE == 1: + req_row = row if MODE == 0 else row // extend_len + real = req_row < bs - num_padding + base = tl.load(seq_lens_ptr + req_row, real, other=0).to(tl.int32) + req = tl.load(req_pool_ptr + req_row, real, other=0).to(tl.int64) + if MODE == 0: + seq_len = tl.where(real, base + seq_offset, 1) + prefix = tl.maximum(seq_len - 1, 0) + else: + seq_len = tl.where(real, base + row % extend_len + 1, 1) + prefix = tl.where(real, base, 0) + if tl.program_id(1) == 0: + tl.store(row_seq_lens_ptr + row, seq_len) + tl.store(row_req_pool_ptr + row, req.to(tl.int32)) + tl.store(row_prefix_lens_ptr + row, prefix) + else: + seq_len = tl.load(row_seq_lens_ptr + row).to(tl.int32) + req = tl.load(row_req_pool_ptr + row).to(tl.int64) token_row = req * req_to_token_row_stride - current = tl.maximum(seq_len - 1, 0) - last_loc = tl.load(req_to_token_ptr + token_row + current).to(tl.int32) + if tl.program_id(1) == 0: + current = tl.maximum(seq_len - 1, 0) + last_loc = tl.load(req_to_token_ptr + token_row + current).to(tl.int32) - compressed = seq_len // RATIO - tl.store(compressed_lens_ptr + row, compressed) + compressed = seq_len // RATIO + tl.store(compressed_lens_ptr + row, compressed) - # The page-aligned allocator keeps each compression group inside one page, - # so last_loc // RATIO is the group's compressed slot; slot 0 is the padding slot. - boundary = (seq_len > 0) & (seq_len % RATIO == 0) - write_loc = tl.where(boundary, last_loc // RATIO, 0) - tl.store(write_locs_ptr + row, write_loc) + # The page-aligned allocator keeps each compression group inside one page, + # so last_loc // RATIO is the group's compressed slot; slot 0 is the padding slot. + boundary = (seq_len > 0) & (seq_len % RATIO == 0) + write_loc = tl.where(boundary, last_loc // RATIO, 0) + tl.store(write_locs_ptr + row, write_loc) - tl.store(logical_positions_ptr + row, current) - tl.store(state_slots_ptr + row, req * RATIO + (current % RATIO).to(tl.int64)) - ring_base = row.to(tl.int64) * RATIO - for k in tl.static_range(RATIO): - member = tl.maximum(current - (RATIO - 1 - k), 0) - slot = req * RATIO + (member % RATIO).to(tl.int64) - tl.store(ring_locs_ptr + ring_base + k, slot.to(tl.int32)) + tl.store(logical_positions_ptr + row, current) + tl.store(state_slots_ptr + row, req * RATIO + (current % RATIO).to(tl.int64)) + ring_base = row.to(tl.int64) * RATIO + for k in tl.static_range(RATIO): + member = tl.maximum(current - (RATIO - 1 - k), 0) + slot = req * RATIO + (member % RATIO).to(tl.int64) + tl.store(ring_locs_ptr + ring_base + k, slot.to(tl.int32)) # Page-table entries are the request's FULL-KV page ids, read from the # page-aligned req_to_token row; the scoring kernels turn them into @@ -144,13 +141,10 @@ def _qsa_graph_row_metadata_kernel( table_row = page_table_ptr + row.to(tl.int64) * max_pages offs = tl.arange(0, PAGE_BLOCK) row_width_pages = req_to_token_row_stride // FULL_PAGE - for p0 in range(0, max_pages, PAGE_BLOCK): - idx = p0 + offs - valid = idx < tl.minimum(max_pages, row_width_pages) - loc = tl.load( - req_to_token_ptr + token_row + idx * FULL_PAGE, mask=valid, other=0 - ) - tl.store(table_row + idx, tl.maximum(loc // FULL_PAGE, 0), mask=valid) + idx = tl.program_id(1) * PAGE_BLOCK + offs + valid = idx < tl.minimum(max_pages, row_width_pages) + loc = tl.load(req_to_token_ptr + token_row + idx * FULL_PAGE, mask=valid, other=0) + tl.store(table_row + idx, tl.maximum(loc // FULL_PAGE, 0), mask=valid) def supports_graph_metadata_kernels(pool, device) -> bool: @@ -174,6 +168,7 @@ def launch_graph_metadata( metadata, req_to_token, pool, + seq_offset=0, ) -> None: indexer = metadata.indexer_metadata @@ -182,25 +177,24 @@ def launch_graph_metadata( row_req_pool = metadata.row_req_pool_indices row_prefix_lens = indexer.graph_prefix_lengths - _qsa_graph_layout_kernel[(bs + 1,)]( - seq_lens, - req_pool_indices, - ( - extend_lens - if extend_lens is not None - else row_seq_lens # unused dummy pointer - ), - row_seq_lens, - row_prefix_lens, - row_req_pool, - bs, - num_rows, - num_padding, - extend_len, - MODE=mode, - num_warps=1, - ) - _qsa_graph_row_metadata_kernel[(num_rows,)]( + if mode == 2: + _qsa_graph_layout_kernel[(bs + 1,)]( + seq_lens, + req_pool_indices, + ( + extend_lens + if extend_lens is not None + else row_seq_lens # unused dummy pointer + ), + row_seq_lens, + row_prefix_lens, + row_req_pool, + bs, + num_rows, + num_padding, + num_warps=1, + ) + _qsa_graph_row_metadata_kernel[(num_rows, triton.cdiv(max_pages, 128))]( row_seq_lens, row_req_pool, indexer.graph_compressed_lengths, @@ -215,5 +209,99 @@ def launch_graph_metadata( RATIO=indexer.compress_ratio, FULL_PAGE=pool.qsa_compressed_page_size * indexer.compress_ratio, PAGE_BLOCK=128, + seq_lens_ptr=seq_lens, + req_pool_ptr=req_pool_indices, + row_prefix_lens_ptr=row_prefix_lens, + bs=bs, + num_padding=num_padding, + extend_len=extend_len, + seq_offset=seq_offset, + MODE=mode, + num_warps=1, + ) + + +@triton.jit +def _qsa_draft_graph_metadata_kernel( + buffers, + seq_lens, + req_pool, + req_to_token, + row_stride, + bs, + num_padding, + MAX_PAGES: tl.constexpr, + RATIO: tl.constexpr, + FULL_PAGE: tl.constexpr, +): + for step in tl.static_range(len(buffers)): + if tl.program_id(2) == step: + buf = buffers[step] + _qsa_graph_row_metadata_kernel( + buf[0], + buf[1], + buf[2], + buf[3], + buf[4], + buf[5], + buf[6], + buf[7], + req_to_token, + row_stride, + MAX_PAGES, + RATIO, + FULL_PAGE, + 128, + seq_lens, + req_pool, + buf[8], + bs, + num_padding, + extend_len=0, + seq_offset=step + 1, + MODE=0, + ) + + +def prepare_draft_graph_metadata(metadata, req_to_token, pool): + buffers = tuple( + ( + m.sequence_lengths, + m.row_req_pool_indices, + m.indexer_metadata.graph_compressed_lengths, + m.indexer_metadata.graph_write_locs, + m.indexer_metadata.graph_compressed_page_table, + m.indexer_metadata.decode_logical_positions, + m.indexer_metadata.pending_ring_slots, + m.indexer_metadata.graph_ring_group_locs, + m.indexer_metadata.graph_prefix_lengths, + ) + for m in metadata + ) + return ( + buffers, + req_to_token, + req_to_token.stride(0), + metadata[0].indexer_metadata.graph_compressed_page_table.shape[1], + pool.qsa_compress_ratio, + pool.qsa_compressed_page_size * pool.qsa_compress_ratio, + ) + + +def launch_draft_graph_metadata(args, seq_lens, req_pool_indices, bs, num_padding): + buffers, req_to_token, row_stride, max_pages, ratio, full_page = args + if bs == 0: + return + _qsa_draft_graph_metadata_kernel[(bs, triton.cdiv(max_pages, 128), len(buffers))]( + buffers, + seq_lens, + req_pool_indices, + req_to_token, + row_stride, + bs, + max(0, min(int(num_padding or 0), bs)), + max_pages, + ratio, + full_page, num_warps=1, ) diff --git a/python/sglang/srt/layers/attention/qwen_sparse_attn_backend.py b/python/sglang/srt/layers/attention/qwen_sparse_attn_backend.py index b6b032dc06..ef398fde8a 100644 --- a/python/sglang/srt/layers/attention/qwen_sparse_attn_backend.py +++ b/python/sglang/srt/layers/attention/qwen_sparse_attn_backend.py @@ -1013,6 +1013,7 @@ class QwenSparseAttnBackend(AttentionBackend): spec_info, seq_lens_cpu, num_padding: int = 0, + seq_offset: int = 0, ) -> None: """Sync-free replay refresh: nothing here may read back to the host, since launch_graph_metadata rebuilds every per-row graph buffer on device.""" @@ -1054,6 +1055,7 @@ class QwenSparseAttnBackend(AttentionBackend): metadata=metadata, req_to_token=self.req_to_token, pool=pool, + seq_offset=seq_offset, ) def _update_qsa_cuda_graph_metadata( @@ -1645,6 +1647,7 @@ class QwenSparseMultiStepDraftBackend: self.model_runner = model_runner self.topk = topk self.speculative_num_steps = speculative_num_steps + self._draft_graph_metadata_args = {} self.attn_backends = [ QwenSparseAttnBackend(model_runner) for _ in range(speculative_num_steps - 1) @@ -1724,11 +1727,13 @@ class QwenSparseMultiStepDraftBackend: ) def init_cuda_graph_state(self, max_bs: int, max_num_tokens: int): + self._draft_graph_metadata_args = {} for backend in self.attn_backends: backend.init_cuda_graph_state(max_bs, max_num_tokens) def init_forward_metadata_out_graph(self, forward_batch, in_capture: bool = False): if in_capture: + self._draft_graph_metadata_args = {} for step, backend in enumerate(self.attn_backends): # Warmup must not write via shared req_pool_idx 0; # pad every capture row to stay below the first compression boundary. @@ -1749,7 +1754,73 @@ class QwenSparseMultiStepDraftBackend: num_padding = getattr(forward_batch, "num_padding", None) num_padding = num_padding if num_padding is not None else 0 + if forward_batch.seq_lens.is_cuda and self.attn_backends: + from sglang.srt.layers.attention.qsa.graph_metadata import ( + launch_draft_graph_metadata, + prepare_draft_graph_metadata, + ) + + bs = forward_batch.batch_size + cached = self._draft_graph_metadata_args.get(bs) + if cached is None: + metadata = tuple( + backend._cuda_graph_metadata[(ForwardMode.DECODE, bs)] + for backend in self.attn_backends + ) + first = self.attn_backends[0] + if all( + backend.req_to_token is first.req_to_token + and backend._can_replay_with_gpu_kernels(m, forward_batch.seq_lens) + and m.indexer_metadata.token_to_kv_pool + is metadata[0].indexer_metadata.token_to_kv_pool + and m.sequence_lengths.numel() == bs + and m.indexer_metadata.graph_compressed_page_table.shape + == metadata[0].indexer_metadata.graph_compressed_page_table.shape + for backend, m in zip(self.attn_backends, metadata) + ): + args = prepare_draft_graph_metadata( + metadata, + first.req_to_token, + metadata[0].indexer_metadata.token_to_kv_pool, + ) + cached = (metadata, args) + self._draft_graph_metadata_args[bs] = cached + if cached is not None: + metadata, args = cached + launch_draft_graph_metadata( + args, + forward_batch.seq_lens, + forward_batch.req_pool_indices, + bs, + num_padding, + ) + for backend, m in zip(self.attn_backends, metadata): + backend.forward_metadata = m + return for step, backend in enumerate(self.attn_backends): + metadata = ( + backend._cuda_graph_metadata[ + (ForwardMode.DECODE, forward_batch.batch_size) + ] + if forward_batch.seq_lens.is_cuda + else None + ) + if metadata is not None and backend._can_replay_with_gpu_kernels( + metadata, forward_batch.seq_lens + ): + backend._replay_cuda_graph_metadata_gpu( + metadata, + bs=forward_batch.batch_size, + req_pool_indices=forward_batch.req_pool_indices, + seq_lens=forward_batch.seq_lens, + forward_mode=ForwardMode.DECODE, + spec_info=forward_batch.spec_info, + seq_lens_cpu=forward_batch.seq_lens_cpu, + num_padding=num_padding, + seq_offset=step + 1, + ) + backend.forward_metadata = metadata + continue step_batch = self._make_step_forward_batch( forward_batch, step, num_padding=num_padding ) diff --git a/test/registered/kernels/ops/attention/qsa/test_qsa.py b/test/registered/kernels/ops/attention/qsa/test_qsa.py index 6038bae1ab..530dfb662c 100644 --- a/test/registered/kernels/ops/attention/qsa/test_qsa.py +++ b/test/registered/kernels/ops/attention/qsa/test_qsa.py @@ -1245,7 +1245,8 @@ def test_qsa_mtp_step_out_cache_loc_matches_draft_forward_layout(): assert torch.equal(backend._step_out_cache_loc(fb, 0), flat) -def test_qsa_graph_metadata_kernels_match_legacy_host_path(): +@pytest.mark.parametrize("max_pages", [32, 129, 2048, 8193]) +def test_qsa_graph_metadata_kernels_match_legacy_host_path(max_pages): """For decode rows and target-verify fan-out (boundary and non-boundary), replay kernels and the host refresh must build identical graph buffers.""" from sglang.srt.layers.attention.qsa.graph_metadata import launch_graph_metadata @@ -1263,11 +1264,13 @@ def test_qsa_graph_metadata_kernels_match_legacy_host_path(): pool = _Pool() backend = QwenSparseAttnBackend.__new__(QwenSparseAttnBackend) # A page-aligned token table: request r's token i lives in full page - # (r * 32 + i // 64) at offset i % 64, mirroring the paged allocator. + # (r * max_pages + i // 64) at offset i % 64, mirroring the paged allocator. rows = torch.arange(8, dtype=torch.int32, device=device)[:, None] - cols = torch.arange(2048, dtype=torch.int32, device=device)[None, :] + cols = torch.arange(max_pages * full_page, dtype=torch.int32, device=device)[ + None, : + ] backend.req_to_token = ( - (rows * 32 + cols // full_page) * full_page + cols % full_page + (rows * max_pages + cols // full_page) * full_page + cols % full_page ).contiguous() seq_lens = torch.tensor(seq_lens_list, dtype=torch.int32, device=device) req_pool_indices = torch.arange(bs, dtype=torch.int32, device=device) @@ -1295,7 +1298,7 @@ def test_qsa_graph_metadata_kernels_match_legacy_host_path(): num_rows, dtype=torch.int32, device=device ), graph_compressed_page_table=torch.zeros( - (num_rows, 32), dtype=torch.int32, device=device + (num_rows, max_pages), dtype=torch.int32, device=device ), graph_compressed_lengths=torch.zeros( num_rows, dtype=torch.int32, device=device @@ -1386,9 +1389,10 @@ def _qsa_expected_graph_layout( for pid in range(bs): base = seq_lens[pid] if mode == 0: + base = base if pid < real_reqs else 1 row_lens.append(base) row_prefix.append(max(base - 1, 0)) - row_reqs.append(req_pool[pid]) + row_reqs.append(req_pool[pid] if pid < real_reqs else 0) continue eff = ( (extend_len if pid < real_reqs else 0) @@ -1409,7 +1413,8 @@ def _qsa_expected_graph_layout( return row_lens, row_prefix, row_reqs -def test_qsa_graph_layout_covers_speculative_rows_and_padded_tail(): +@pytest.mark.parametrize("seq_offset", [0, 1, 3]) +def test_qsa_graph_layout_covers_speculative_rows_and_padded_tail(seq_offset): """The layout kernel must rebuild speculative row fan-out and the padded dummy tail; the row-metadata kernel must derive compressed slots from those rows.""" from sglang.srt.layers.attention.qsa.graph_metadata import launch_graph_metadata @@ -1488,6 +1493,7 @@ def test_qsa_graph_layout_covers_speculative_rows_and_padded_tail(): metadata=metadata, req_to_token=backend.req_to_token, pool=pool, + seq_offset=seq_offset if mode == 0 else 0, ) torch.cuda.synchronize() @@ -1495,7 +1501,7 @@ def test_qsa_graph_layout_covers_speculative_rows_and_padded_tail(): mode=mode, bs=bs, num_rows=num_rows, - seq_lens=seq_lens, + seq_lens=[x + seq_offset for x in seq_lens] if mode == 0 else seq_lens, req_pool=req_pool, extend_lens=extend_lens, extend_len=extend_len, @@ -1534,17 +1540,105 @@ def test_qsa_graph_layout_covers_speculative_rows_and_padded_tail(): extend_len=0, num_padding=2, ) + run_case( + mode=0, + bs=4, + num_rows=4, + seq_lens=[255, 1024, 4096, 4096], + extend_lens=None, + extend_len=0, + num_padding=2, + ) # Decode with a padded tail (dummy rows alias request slot 0). run_case( mode=0, bs=4, num_rows=4, - seq_lens=[256, 1024, 1025, 4096], + seq_lens=[256, 1024, 1025, 4080], extend_lens=None, extend_len=0, num_padding=0, ) +@pytest.mark.parametrize("bs", [1, 3, 128]) +@pytest.mark.parametrize("padding", [0, 1]) +def test_qsa_draft_metadata_multi_step_graph(bs, padding): + from types import SimpleNamespace + + from sglang.srt.layers.attention.qsa.graph_metadata import ( + launch_draft_graph_metadata, + launch_graph_metadata, + prepare_draft_graph_metadata, + ) + + pages, ratio, full_page = 129, 4, 64 + pool = SimpleNamespace(qsa_compress_ratio=ratio, qsa_compressed_page_size=16) + table = torch.arange( + (bs + 2) * pages * full_page, dtype=torch.int32, device="cuda" + ).reshape(bs + 2, -1) + seq = torch.full((bs,), 8190, dtype=torch.int64, device="cuda") + reqs = torch.arange(1, bs + 1, dtype=torch.int32, device="cuda") + + def metadata(): + indexer = SimpleNamespace( + graph_compressed_lengths=torch.empty(bs, dtype=torch.int32, device="cuda"), + graph_write_locs=torch.empty(bs, dtype=torch.int32, device="cuda"), + graph_compressed_page_table=torch.empty( + (bs, pages), dtype=torch.int32, device="cuda" + ), + decode_logical_positions=torch.empty(bs, dtype=torch.int32, device="cuda"), + pending_ring_slots=torch.empty(bs, dtype=torch.int64, device="cuda"), + graph_ring_group_locs=torch.empty( + (bs, ratio), dtype=torch.int32, device="cuda" + ), + graph_prefix_lengths=torch.empty(bs, dtype=torch.int32, device="cuda"), + compress_ratio=ratio, + ) + return SimpleNamespace( + sequence_lengths=torch.empty(bs, dtype=torch.int32, device="cuda"), + row_req_pool_indices=torch.empty(bs, dtype=torch.int32, device="cuda"), + indexer_metadata=indexer, + ) + + outputs = [metadata() for _ in range(3)] + references = [metadata() for _ in range(3)] + args = prepare_draft_graph_metadata(outputs, table, pool) + launch_draft_graph_metadata(args, seq, reqs, bs, padding) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + launch_draft_graph_metadata(args, seq, reqs, bs, padding) + for replay in range(3): + seq.add_(replay) + reqs.copy_(reqs.roll(1)) + graph.replay() + for step, (out, ref) in enumerate(zip(outputs, references)): + launch_graph_metadata( + mode=0, + bs=bs, + num_rows=bs, + seq_lens=seq, + req_pool_indices=reqs, + extend_lens=None, + extend_len=0, + num_padding=padding, + metadata=ref, + req_to_token=table, + pool=pool, + seq_offset=step + 1, + ) + torch.testing.assert_close( + out.sequence_lengths, ref.sequence_lengths, rtol=0, atol=0 + ) + torch.testing.assert_close( + out.row_req_pool_indices, ref.row_req_pool_indices, rtol=0, atol=0 + ) + for key, value in vars(out.indexer_metadata).items(): + if isinstance(value, torch.Tensor): + torch.testing.assert_close( + value, getattr(ref.indexer_metadata, key), rtol=0, atol=0 + ) + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"]))