diff --git a/python/sglang/kernels/ops/attention/metadata.py b/python/sglang/kernels/ops/attention/metadata.py index f995f9ea3d..3ded6383b1 100644 --- a/python/sglang/kernels/ops/attention/metadata.py +++ b/python/sglang/kernels/ops/attention/metadata.py @@ -164,15 +164,27 @@ def _resolve_metadata_page_ids( token_ids, mask, PAGE_SIZE: tl.constexpr, + page_mapping, + page_mapping_stride, token_mapping=None, token_mapping_stride=0, ): - # Static SWA maps token slots. An absent mapping compiles to an identity. + # Static SWA maps token slots; unified pools map pages. Callers supply + # the appropriate mapping, and absent mappings compile to an identity. if token_mapping is not None: token_ids = tl.load( token_mapping + token_ids * token_mapping_stride, mask=mask, other=0 ) page_ids = token_ids // PAGE_SIZE + if page_mapping is not None: + page_ids = tl.maximum( + tl.load( + page_mapping + tl.maximum(page_ids, 0) * page_mapping_stride, + mask=mask, + other=0, + ), + 0, + ) return page_ids @@ -197,12 +209,17 @@ def _fused_decode_metadata_kernel( swa_page_table_stride_1, full_to_swa_index_mapping, full_to_swa_mapping_stride_0, + full_v2p, + full_v2p_stride_0, + swa_v2p, + swa_v2p_stride_0, B, max_seq_pages, PAGE_SIZE: tl.constexpr, SEQ_LEN_DELTA: tl.constexpr, BLOCK_COLS: tl.constexpr, SKIP_PAGE_TABLE: tl.constexpr, + ZERO_TAIL: tl.constexpr, ): pid_b = tl.program_id(0) pid_c = tl.program_id(1) @@ -228,7 +245,7 @@ def _fused_decode_metadata_kernel( tl.cdiv(seq_len + SEQ_LEN_DELTA, PAGE_SIZE), max_seq_pages ) col_start = pid_c * BLOCK_COLS - if col_start >= num_live_pages: + if not ZERO_TAIL and col_start >= num_live_pages: return row_idx = tl.load(req_pool_indices + pid_b * req_pool_indices_stride_0) @@ -242,11 +259,14 @@ def _fused_decode_metadata_kernel( other=0, cache_modifier=".cg", ) - full_pages = _resolve_metadata_page_ids(token_ids, live_mask, PAGE_SIZE) + store_mask = cols < max_seq_pages if ZERO_TAIL else live_mask + full_pages = _resolve_metadata_page_ids( + token_ids, live_mask, PAGE_SIZE, full_v2p, full_v2p_stride_0 + ) tl.store( page_table + pid_b * page_table_stride_0 + cols * page_table_stride_1, full_pages, - mask=live_mask, + mask=store_mask, cache_modifier=".cg", ) if swa_page_table is not None: @@ -254,6 +274,8 @@ def _fused_decode_metadata_kernel( token_ids, live_mask, PAGE_SIZE, + swa_v2p, + swa_v2p_stride_0, full_to_swa_index_mapping, full_to_swa_mapping_stride_0, ) @@ -262,7 +284,7 @@ def _fused_decode_metadata_kernel( + pid_b * swa_page_table_stride_0 + cols * swa_page_table_stride_1, swa_pages, - mask=live_mask, + mask=store_mask, cache_modifier=".cg", ) @@ -490,30 +512,39 @@ def normal_decode_set_metadata( swa_page_table: Optional[torch.Tensor] = None, full_to_swa_index_mapping: Optional[torch.Tensor] = None, skip_page_table: bool = False, + full_v2p: Optional[torch.Tensor] = None, + swa_v2p: Optional[torch.Tensor] = None, + zero_tail: bool = False, ): """Fill sequence lengths, their prefix sum and page tables in one launch. - ``req_to_token`` contains token slots. Static SWA uses - ``full_to_swa_index_mapping`` to map FULL token slots to SWA token slots. - Runtime callers resolve this tensor from their pool before calling. + ``req_to_token`` contains token slots. ``full_v2p`` / ``swa_v2p`` map + virtual pages to physical pages; an unbound (-1) page maps to the sink. + Static SWA instead uses ``full_to_swa_index_mapping`` to map token slots. + Supply exactly one SWA mapping when writing ``swa_page_table``. Mappings + are read on device on every invocation, including CUDA graph replay. - Only the live prefix (ceil((seq_len + delta) / page_size)) is rewritten; - consumers must bound reads by ``cache_seqlens_int32``. For pretranslated - tables, ``skip_page_table=True`` leaves both page tables untouched and - computes only lengths and the prefix sum. + Only the live prefix (ceil((seq_len + delta) / page_size)) is rewritten + by default; consumers must bound reads by ``cache_seqlens_int32``. + ``zero_tail=True`` also clears columns up to ``max_seq_pages`` to zero, + independently of address translation. ``skip_page_table=True`` leaves + both page tables untouched and computes only lengths and the prefix sum. """ assert page_size > 0 and (page_size & (page_size - 1)) == 0, ( f"page_size must be a power of two, got {page_size}" ) if swa_page_table is not None and not skip_page_table: - assert full_to_swa_index_mapping is not None, ( - "SWA output requires a token-slot mapping" + assert (full_to_swa_index_mapping is None) != (swa_v2p is None), ( + "SWA output requires exactly one token-slot or page mapping" ) batch_size = cache_seqlens_int32.shape[0] - # Lengths-only/token gathers use 256 columns; page/SWA gathers use 128. + # Lengths-only/unmapped token gathers use 256 columns, other gathers 128. block_cols = ( - 256 if skip_page_table or (page_size == 1 and swa_page_table is None) else 128 + 256 + if skip_page_table + or (page_size == 1 and swa_page_table is None and full_v2p is None) + else 128 ) grid = ( (1, 1) @@ -542,12 +573,17 @@ def normal_decode_set_metadata( full_to_swa_index_mapping.stride(0) if full_to_swa_index_mapping is not None else 0, + full_v2p, + full_v2p.stride(0) if full_v2p is not None else 0, + swa_v2p, + swa_v2p.stride(0) if swa_v2p is not None else 0, batch_size, max_seq_pages, PAGE_SIZE=page_size, SEQ_LEN_DELTA=seq_len_delta, BLOCK_COLS=block_cols, SKIP_PAGE_TABLE=skip_page_table, + ZERO_TAIL=zero_tail, num_warps=block_cols // 32, num_stages=3, ) diff --git a/python/sglang/srt/arg_groups/kv_cache_hook.py b/python/sglang/srt/arg_groups/kv_cache_hook.py index 24c6e1a001..2d4ab9a5ee 100644 --- a/python/sglang/srt/arg_groups/kv_cache_hook.py +++ b/python/sglang/srt/arg_groups/kv_cache_hook.py @@ -488,10 +488,14 @@ _SPEC_VERIFY_AUDITED_BACKENDS = frozenset( } ) +_SPEC_MHA_AUDITED_BACKENDS = frozenset({"triton", "flashinfer", "fa3", "trtllm_mha"}) + # The MHA backends that read through the KV-index translator on every path a # draft forward takes. A fused draft region holds dense K/V rows, so a fused # draft runs only on these. -TRANSLATED_MHA_RAILS = frozenset({"triton", "flashinfer", "fa3", "trtllm_mha"}) +# FA4 carries translated draft ids, but speculative admission is audited +# per algorithm below; currently only DSPARK is validated with FA4. +TRANSLATED_MHA_RAILS = _SPEC_MHA_AUDITED_BACKENDS | {"fa4"} def _assert_spec_verify_backends( @@ -613,7 +617,7 @@ def handle_unified_memory_pool(server_args: Any) -> None: eagle_allowed = ( _SPEC_VERIFY_AUDITED_BACKENDS if use_mla_backend(server_args) - else TRANSLATED_MHA_RAILS + else _SPEC_MHA_AUDITED_BACKENDS ) eagle_backends = set(attention_backends_of(resolved_view(server_args))) assert ( @@ -628,20 +632,24 @@ def handle_unified_memory_pool(server_args: Any) -> None: # An unset draft backend inherits the target's pair. draft_backend = cfg.speculative_draft_attention_backend draft_backends = {draft_backend} if draft_backend else eagle_backends - assert draft_backends <= TRANSLATED_MHA_RAILS, ( + assert draft_backends <= _SPEC_MHA_AUDITED_BACKENDS, ( "--enable-unified-memory + EAGLE/EAGLE3 runs the draft on " f"{sorted(draft_backends)}, but a fused draft is MHA-shaped and " "reads its KV through the translated MHA rails " - f"{sorted(TRANSLATED_MHA_RAILS)}. " + f"{sorted(_SPEC_MHA_AUDITED_BACKENDS)}. " "Set --speculative-draft-attention-backend to one of them." ) if cfg.speculative_algorithm == "DSPARK": - _assert_spec_verify_backends(server_args, algorithm="DSPARK") + _assert_spec_verify_backends( + server_args, + algorithm="DSPARK", + allowed=_SPEC_VERIFY_AUDITED_BACKENDS | {"fa4"}, + ) if cfg.speculative_algorithm == "DFLASH": _assert_spec_verify_backends( server_args, algorithm="DFLASH", - allowed=TRANSLATED_MHA_RAILS, + allowed=_SPEC_MHA_AUDITED_BACKENDS, ) assert not cfg.enable_two_batch_overlap, ( "--enable-unified-memory does not support --enable-two-batch-overlap: " diff --git a/python/sglang/srt/layers/attention/flashattention_backend.py b/python/sglang/srt/layers/attention/flashattention_backend.py index 4f6ed47abf..be7088d7af 100644 --- a/python/sglang/srt/layers/attention/flashattention_backend.py +++ b/python/sglang/srt/layers/attention/flashattention_backend.py @@ -463,7 +463,39 @@ class FlashAttentionBackend(AttentionBackend): k_sf, v_sf = self.token_to_kv_pool.get_kv_scale_buffer(layer.layer_id) return {"sfq": q_descale, "sfk": k_sf, "sfv": v_sf} + def _capture_unified_verify_metadata(self, forward_batch: ForwardBatch) -> bool: + # Fixed-width verify uses only capture-stable request/length buffers. + # Ragged layouts and encoder offsets still use their eager builders. + return ( + forward_batch.forward_mode.is_target_verify() + and self.fa_impl_ver == 4 + and self.kv_index_translator.reads_are_translated + and self.topk <= 1 + and forward_batch.encoder_lens is None + and getattr(forward_batch.spec_info, "ragged_verify_layout", None) is None + ) + def init_forward_metadata_in_graph(self, forward_batch: ForwardBatch) -> None: + if self._capture_unified_verify_metadata(forward_batch): + metadata = self.target_verify_metadata[forward_batch.batch_size] + translator = self.kv_index_translator + swa = translator.space(IdSpaceKind.SLIDING_WINDOW) + normal_decode_set_metadata( + metadata.cache_seqlens_int32, + metadata.cu_seqlens_k, + metadata.page_table, + self.req_to_token, + forward_batch.req_pool_indices, + self.max_num_pages, + forward_batch.seq_lens, + self.speculative_num_draft_tokens, + self.page_size, + metadata.swa_page_table, + full_v2p=translator.space(IdSpaceKind.FULL).read_v2p, + swa_v2p=None if swa is None else swa.read_v2p, + zero_tail=True, + ) + return # Single-CG has no Python between steps, so one capturable kernel updates # the persistent metadata. if not forward_batch.forward_mode.is_draft_extend_v2(): @@ -594,6 +626,23 @@ class FlashAttentionBackend(AttentionBackend): else: self.cuda_graph_swa_out_cache_loc[:n].copy_(swa_out_cache_loc) + if self._capture_unified_verify_metadata(forward_batch): + if in_capture: + self._bind_metadata_buffers( + bs, + forward_batch.positions.numel(), + encoder_lens, + forward_mode, + spec_info, + seq_lens.device, + ) + self.forward_metadata = self.target_verify_metadata[bs] + self.forward_metadata_spec_decode_expand = None + # The runner records the device-side update at the start of the + # model graph. Do not populate a live plan's read-table cache with + # this graph-owned buffer: another bucket can overwrite it later. + return + if in_capture: num_tokens = forward_batch.positions.numel() seq_lens_cpu = seq_lens.cpu() diff --git a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py index ed3ec3c64a..fb837274ea 100644 --- a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py @@ -1258,16 +1258,11 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): attn_backend.init_forward_metadata_out_graph(forward_batch, in_capture=True) def run_once(): - # Graph-recordable metadata-prep hook. The unified memory pool - # records ZERO translate nodes here: all its read/write translates - # run eagerly in `init_forward_metadata_out_graph` (replay-prep), so - # the captured graph reads already-physical locs. Base no-op for triton. + # Backends may gather live pool mappings into their static + # metadata here; these nodes run before any model KV reads. attn_backend.init_forward_metadata_in_graph(forward_batch) self._record_in_graph_metadata_prep_done() - # No invalidate_loc_cache() here: the unified pool translates its - # locs in `init_forward_metadata_out_graph`, so no cache to invalidate. - forward_batch.dp_local_start_pos = forward_batch.dp_local_num_tokens = ( None ) diff --git a/test/registered/kernels/ops/attention/test_decode_metadata_mapping.py b/test/registered/kernels/ops/attention/test_decode_metadata_mapping.py index bb358dbe2e..674f327d82 100644 --- a/test/registered/kernels/ops/attention/test_decode_metadata_mapping.py +++ b/test/registered/kernels/ops/attention/test_decode_metadata_mapping.py @@ -1,4 +1,4 @@ -"""Decode metadata must preserve strided storage and untouched row tails.""" +"""Decode page addressing and row-tail policy are independent contracts.""" import itertools import unittest @@ -9,17 +9,23 @@ from sglang.kernels.ops.attention.metadata import normal_decode_set_metadata from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import CustomTestCase -register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="1-gpu-large") +register_cuda_ci(est_time=15, stage="base-b-kernel-unit", runner_config="1-gpu-large") @unittest.skipUnless(torch.cuda.is_available(), "requires CUDA") class TestDecodeMetadataMapping(CustomTestCase): - def test_static_mapping_and_strides(self): - for ps, use_swa in itertools.product((1, 128), (False, True)): - with self.subTest(ps=ps, use_swa=use_swa): - self._check_tables(ps, use_swa) + def test_mapping_and_tail_policy(self): + # Cross a whole column block of dead pages, so zero_tail must also + # control the early exit rather than just the last block's store mask. + for ps, mapped_full, swa_kind, zero_tail in itertools.product( + (1, 128), (False, True), (None, "token", "page"), (False, True) + ): + with self.subTest( + ps=ps, mapped_full=mapped_full, swa=swa_kind, zero_tail=zero_tail + ): + self._check_tables(ps, mapped_full, swa_kind, zero_tail) - def _check_tables(self, ps, use_swa): + def _check_tables(self, ps, mapped_full, swa_kind, zero_tail): device = "cuda" bs, width, delta, sentinel = 3, 300, 3, -17 # Use strided inputs/outputs and mappings: graph replay must read the @@ -32,16 +38,22 @@ class TestDecodeMetadataMapping(CustomTestCase): rows = torch.tensor([1, 9, 2, 9, 0, 9], device=device)[::2] lengths = torch.tensor([2 * ps + 1, 1, 0], device=device, dtype=torch.int32) seq_lens = torch.stack((lengths - delta, lengths), dim=1)[:, 0] + full = ((torch.arange(64, device=device, dtype=torch.int32) * 7) % 31)[::2] + swa = ((torch.arange(64, device=device, dtype=torch.int32) * 11) % 31)[::2] + full[21] = -1 + swa[22] = -1 slots = torch.arange(32 * ps, device=device) token_map = ((slots // ps * 3 + 5) % 31) * ps + slots % ps token_map = torch.stack((token_map, token_map), dim=1)[:, 0] - token_map = token_map if use_swa else None + full_map = full if mapped_full else None + swa_map = swa if swa_kind == "page" else None + token_map = token_map if swa_kind == "token" else None full_storage = torch.full( (bs, 2 * (width + 2)), sentinel, device=device, dtype=torch.int32 ) swa_storage = torch.full_like(full_storage, sentinel) page_table = full_storage[:, ::2] - swa_table = swa_storage[:, ::2] if use_swa else None + swa_table = swa_storage[:, ::2] if swa_kind is not None else None cache_lens = torch.empty_like(lengths) cu_lens = torch.empty(bs + 1, device=device, dtype=torch.int32) @@ -57,6 +69,9 @@ class TestDecodeMetadataMapping(CustomTestCase): ps, swa_page_table=swa_table, full_to_swa_index_mapping=token_map, + full_v2p=full_map, + swa_v2p=swa_map, + zero_tail=zero_tail, ) torch.testing.assert_close(cache_lens, lengths) @@ -68,7 +83,13 @@ class TestDecodeMetadataMapping(CustomTestCase): < ((lengths + ps - 1) // ps)[:, None] ) expected_full = gathered // ps - expected_swa = token_map[gathered] // ps if use_swa else None + if mapped_full: + expected_full = full[expected_full].clamp_min(0) + expected_swa = None + if swa_kind == "token": + expected_swa = token_map[gathered] // ps + elif swa_kind == "page": + expected_swa = swa[gathered // ps].clamp_min(0) for storage, expected in ( (full_storage, expected_full), (swa_storage, expected_swa), @@ -76,7 +97,9 @@ class TestDecodeMetadataMapping(CustomTestCase): if expected is None: continue want = torch.full_like(storage, sentinel) - want[:, : 2 * width : 2] = torch.where(live, expected, sentinel) + want[:, : 2 * width : 2] = torch.where( + live, expected, 0 if zero_tail else sentinel + ) torch.testing.assert_close(storage, want) def test_lengths_only_leaves_tables_untouched(self): @@ -100,6 +123,7 @@ class TestDecodeMetadataMapping(CustomTestCase): swa_page_table=swa_table, full_to_swa_index_mapping=torch.arange(4 * ps, device="cuda"), skip_page_table=skip, + zero_tail=True, ) torch.testing.assert_close(cache_lens, (seq_lens + 3).int()) torch.testing.assert_close( diff --git a/test/registered/unit/layers/attention/test_flashattention_graph_metadata.py b/test/registered/unit/layers/attention/test_flashattention_graph_metadata.py index 0e63e94f6b..956f2beb93 100644 --- a/test/registered/unit/layers/attention/test_flashattention_graph_metadata.py +++ b/test/registered/unit/layers/attention/test_flashattention_graph_metadata.py @@ -5,11 +5,12 @@ import torch from sglang.srt.layers.attention.flashattention_backend import FlashAttentionBackend from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator +from sglang.srt.mem_cache.kv_loc_plan import IdSpace, IdSpaceKind from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import CustomTestCase -register_cuda_ci(est_time=10, stage="base-b", runner_config="1-gpu-small") +register_cuda_ci(est_time=15, stage="base-b", runner_config="1-gpu-small") class TestFlashAttentionGraphMetadata(CustomTestCase): @@ -141,5 +142,129 @@ class TestDraftExtendInGraph(CustomTestCase): self._run(translating=False) +@unittest.skipUnless(torch.cuda.is_available(), "CUDA graph replay requires CUDA") +class TestUnifiedVerifyGraphMetadata(CustomTestCase): + def test_replay_uses_live_mapping_and_request_rows(self): + """Captured tables must follow compaction, page reuse and shorter rows. + + A raw req_to_token gather silently uses virtual pages; an eager-only + builder leaves stale physical pages after replay. Both violate the + expected physical addresses checked after each mapping change. + """ + for page_size in (1, 128): + for use_swa in (False, True): + with self.subTest(page_size=page_size, use_swa=use_swa): + self._check_replay(page_size, use_swa) + + def _check_replay(self, ps, use_swa): + device = "cuda" + bs, pages, delta = 3, 6, 3 + b = FlashAttentionBackend.__new__(FlashAttentionBackend) + b.fa_impl_ver = 4 + b.topk = 1 + b.page_size = ps + b.max_num_pages = pages + b.speculative_num_draft_tokens = delta + b.use_sliding_window_kv_pool = use_swa + # Request zero is the padded sink. Live virtual pages differ from + # physical pages, including an unbound SWA page (-1). + b.req_to_token = torch.zeros((4, pages * ps), dtype=torch.int64, device=device) + for row in range(1, 4): + b.req_to_token[row] = torch.arange( + row * pages * ps, (row + 1) * pages * ps, device=device + ) + full = torch.arange(24, device=device, dtype=torch.int32).flip(0).contiguous() + full[0] = 0 + swa = torch.arange(24, device=device, dtype=torch.int32) + swa[7] = -1 + translator = KVIndexTranslator.__new__(KVIndexTranslator) + translator._spaces = { + IdSpaceKind.FULL: IdSpace(key="full", read_v2p=full), + } + if use_swa: + translator._spaces[IdSpaceKind.SLIDING_WINDOW] = IdSpace( + key="swa", read_v2p=swa + ) + b.kv_index_translator = translator + b.target_verify_metadata = { + "cache_seqlens": torch.zeros(bs, dtype=torch.int32, device=device), + "cu_seqlens_k": torch.zeros(bs + 1, dtype=torch.int32, device=device), + "cu_seqlens_q": torch.arange( + 0, (bs + 1) * delta, delta, dtype=torch.int32, device=device + ), + "page_table": torch.full( + (bs, pages), -99, dtype=torch.int32, device=device + ), + } + if use_swa: + b.target_verify_metadata["swa_page_table"] = torch.full( + (bs, pages), -99, dtype=torch.int32, device=device + ) + b.cuda_graph_swa_out_cache_loc = torch.zeros( + bs * delta, dtype=torch.int64, device=device + ) + fb = SimpleNamespace( + forward_mode=ForwardMode.TARGET_VERIFY, + batch_size=bs, + req_pool_indices=torch.tensor([2, 1, 0], device=device), + seq_lens=torch.tensor([ps + 1, 2 * ps - 1, 1], device=device), + encoder_lens=None, + spec_info=SimpleNamespace(ragged_verify_layout=None), + positions=torch.arange(bs * delta, device=device), + out_cache_loc=None, + ) + b.init_forward_metadata_out_graph(fb, in_capture=True) + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + b.init_forward_metadata_in_graph(fb) + torch.cuda.current_stream().wait_stream(stream) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=stream): + b.init_forward_metadata_in_graph(fb) + metadata = b.target_verify_metadata[bs] + pointer = metadata.page_table.data_ptr() + for iteration in range(3): + if iteration == 1: + # Compaction and request reorder, plus a shorter live prefix. + full[6:24] = torch.arange(1, 19, device=device, dtype=torch.int32) + swa[7] = 21 + swa[13] = -1 + fb.req_pool_indices.copy_(torch.tensor([1, 3, 0], device=device)) + fb.seq_lens.copy_(torch.tensor([0, 3 * ps, 1], device=device)) + elif iteration == 2: + # Recycle a request row, grow its extent and reuse its pages. + b.req_to_token[1].copy_(b.req_to_token[2]) + full[12:18] = 4 + fb.seq_lens.copy_(torch.tensor([2 * ps, 0, 1], device=device)) + b.init_forward_metadata_out_graph(fb) + graph.replay() + torch.cuda.synchronize() + self.assertEqual(metadata.page_table.data_ptr(), pointer) + lengths = fb.seq_lens.cpu() + delta + torch.testing.assert_close( + metadata.cache_seqlens_int32.cpu(), lengths.int() + ) + torch.testing.assert_close( + metadata.cu_seqlens_k.cpu(), + torch.cat((torch.zeros(1, dtype=torch.int32), lengths.cumsum(0).int())), + ) + rows = fb.req_pool_indices.cpu().tolist() + for table, mapping in ( + (metadata.page_table, full), + (metadata.swa_page_table, swa), + ): + if table is None: + continue + expected = torch.zeros((bs, pages), dtype=torch.int32) + tokens = b.req_to_token.cpu() + physical = mapping.cpu() + for lane, row in enumerate(rows): + for page in range(min(pages, (int(lengths[lane]) + ps - 1) // ps)): + virtual = int(tokens[row, page * ps]) // ps + expected[lane, page] = max(0, int(physical[virtual])) + torch.testing.assert_close(table.cpu(), expected) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/mem_cache/test_fused_draft_placement.py b/test/registered/unit/mem_cache/test_fused_draft_placement.py index fea2a14eda..aa0735cf31 100644 --- a/test/registered/unit/mem_cache/test_fused_draft_placement.py +++ b/test/registered/unit/mem_cache/test_fused_draft_placement.py @@ -72,7 +72,7 @@ from sglang.srt.runtime_context import get_parallel, override_platform from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase -register_cpu_ci(est_time=5, suite="base-a-test-cpu") +register_cpu_ci(est_time=15, suite="base-a-test-cpu") _HOSTS = ("full", "swa") _HOSTS_TRI = ("full", "swa", "mamba") @@ -744,17 +744,17 @@ class TestFusedDraftDecision(CustomTestCase): def test_a_draft_off_the_translated_rails_keeps_the_private_pool(self): """A fused draft reads its rows through the KV-index translator, which - only triton, flashinfer, fa3 and trtllm_mha carry on every draft path. + triton, flashinfer, fa3, fa4 and trtllm_mha carry on draft paths. The draft's backend is the published one, a model hook's declaration included; otherwise it is what the draft runner inherits from the target.""" - declined = self._decide(algorithm="DSPARK", draft_backend="fa4") + declined = self._decide(algorithm="EAGLE", draft_backend="flashmla") self.assertIsNone(declined.placement) - self.assertIn("fa4", declined.declined) + self.assertIn("flashmla", declined.declined) # A DFLASH-family draft runs the target's prefill backend, or the # platform default when that is not a draft backend. for target_backends, fuses in ( - (("fa4", "fa4"), False), + (("fa4", "fa4"), True), (("fa3", "flashmla"), True), (("flashmla", "flashmla"), True), ): @@ -763,6 +763,10 @@ class TestFusedDraftDecision(CustomTestCase): # An EAGLE draft with no backend of its own runs the target's pair. self.assertIsNone(self._decide(target_backends=("fa3", "flashmla")).placement) + def test_a_fa4_dspark_draft_fuses(self): + decision = self._decide(algorithm="DSPARK", draft_backend="fa4") + self.assertIsNotNone(decision.placement) + def test_a_trtllm_mha_draft_fuses(self): """trtllm_mha refills its graph page table to what each replay reads and widens its eager table by the draft block, so its draft fuses.""" diff --git a/test/registered/unit/server_args/test_unified_memory_spec_gate.py b/test/registered/unit/server_args/test_unified_memory_spec_gate.py index bc3aa24674..d4195df061 100644 --- a/test/registered/unit/server_args/test_unified_memory_spec_gate.py +++ b/test/registered/unit/server_args/test_unified_memory_spec_gate.py @@ -31,8 +31,9 @@ from sglang.srt.arg_groups.kv_cache_hook import handle_unified_memory_pool from sglang.srt.configs.model_config import AttentionArch from sglang.srt.server_args import ServerArgs from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase -register_cpu_ci(est_time=4, suite="base-a-test-cpu") +register_cpu_ci(est_time=15, suite="base-a-test-cpu") class _PlainHFConfig: @@ -100,17 +101,18 @@ _VERIFY_BACKENDS = ( _MHA_RAILS = ("triton", "flashinfer", "fa3", "trtllm_mha") -class TestUnifiedMemorySpecGate(unittest.TestCase): +class TestUnifiedMemorySpecGate(CustomTestCase): VERIFY_BACKENDS = _VERIFY_BACKENDS MHA_RAILS = _MHA_RAILS MLA_ONLY = tuple(b for b in _VERIFY_BACKENDS if b not in _MHA_RAILS) UNAUDITED = ("fa4",) def test_dspark_verify_backends(self): - for backend in self.VERIFY_BACKENDS: + for backend in self.VERIFY_BACKENDS + ("fa4",): self.assertTrue(_accepts("DSPARK", backend=backend), backend) + self.assertTrue(_accepts("DSPARK", backend="fa4", draft_backend="fa4")) # trtllm_mha is audited for the EAGLE and DFLASH arms only. - for backend in self.UNAUDITED + ("trtllm_mha",): + for backend in ("trtllm_mha",): self.assertFalse(_accepts("DSPARK", backend=backend), backend) def test_dcp_keeps_flashinfer_off_the_verify_list(self):