diff --git a/python/sglang/srt/layers/attention/linear/kda_backend.py b/python/sglang/srt/layers/attention/linear/kda_backend.py index 29cb57b9aa..d54eb1d9f1 100644 --- a/python/sglang/srt/layers/attention/linear/kda_backend.py +++ b/python/sglang/srt/layers/attention/linear/kda_backend.py @@ -624,8 +624,15 @@ class KDAAttnBackend(MambaAttnBackendBase): def supports_prefill_graph_extend(self) -> bool: return ( not envs.SGLANG_DISABLE_KDA_PREFILL_GRAPH_EXTEND.get() - # Only the Triton chain guards padded rows (-1 state slots, chunks past T). - and isinstance(self.kernel_dispatcher.extend_kernel, TritonKDAKernel) + and ( + isinstance(self.kernel_dispatcher.extend_kernel, TritonKDAKernel) + or getattr( + self.kernel_dispatcher.extend_kernel, + "prefill_graph_wrapper_cls", + None, + ) + is not None + ) # Fused-accept staging (flashinfer verify) writes per extend. and self.accept_lens_pool is None ) @@ -638,13 +645,17 @@ class KDAAttnBackend(MambaAttnBackendBase): ) -> KDAPrefillGraphMetadata: cache_indices = self._prefill_graph_cache_indices(forward_batch) meta = KDAPrefillGraphMetadata.allocate( - num_tokens=forward_batch.positions.shape[0], + # KDA sees global sequence order, even when CP positions are local. + num_tokens=forward_batch.extend_num_tokens, max_seqs_cap=self.prefill_graph_max_seqs, # Any replay of this bucket may carry prefix-cache snapshots (the # extra buffer is only on with the radix cache). track=get_exec().mamba.enable_mamba_extra_buffer, cache_index_dtype=cache_indices.dtype, device=self.device, + flashinfer_wrapper_cls=getattr( + self.kernel_dispatcher.extend_kernel, "prefill_graph_wrapper_cls", None + ), ) self._fill_prefill_graph_metadata( meta=meta, forward_batch=forward_batch, cache_indices=cache_indices diff --git a/python/sglang/srt/layers/attention/linear/kda_flashinfer_prefill_graph.py b/python/sglang/srt/layers/attention/linear/kda_flashinfer_prefill_graph.py new file mode 100644 index 0000000000..26197bb7f0 --- /dev/null +++ b/python/sglang/srt/layers/attention/linear/kda_flashinfer_prefill_graph.py @@ -0,0 +1,88 @@ +"""Fixed-capacity FlashInfer KDA plans for breakable prefill graphs.""" + +import numpy as np +import torch +import torch.nn.functional as F + + +class FlashInferKDAPrefillGraph: + def __init__(self, *, wrapper_cls, num_tokens, max_seqs, track, device): + self.wrapper_cls = wrapper_cls + # A workspace can be captured only once: each layer owns its plan. + self.wrappers = {} + self.cu_seqlens = torch.empty(max_seqs + 1, dtype=torch.int64, device=device) + self.cache_indices = torch.empty(max_seqs, dtype=torch.int32, device=device) + self.track = track + if track: + self.checkpoint_cu_starts = torch.empty_like(self.cu_seqlens) + self.checkpoint_indices = torch.full( + (num_tokens // 64,), -1, dtype=torch.int32, device=device + ) + + def fill(self, meta, extend_seq_lens, track_rows): + self.cu_seqlens.copy_(meta.cu_seqlens) + # Empty padded sequences may read their initial state, but do not write + # it. FlashInfer does not accept Triton's negative state-slot sentinel. + self.cache_indices.copy_(meta.cache_indices.clamp_min(0)) + if self.track: + counts = np.asarray(extend_seq_lens, dtype=np.int64) // 64 + starts = np.empty(self.cu_seqlens.numel(), dtype=np.int64) + starts[0] = 0 + np.cumsum(counts, out=starts[1 : len(counts) + 1]) + starts[len(counts) + 1 :] = starts[len(counts)] + indices = np.full(self.checkpoint_indices.numel(), -1, dtype=np.int32) + if track_rows is not None: + for row, chunk in track_rows.chunk_rows: + # Triton names the state before chunk; FlashInfer snapshots + # after each interval. Write that snapshot directly to row. + indices[starts[row] + chunk - 1] = row + self.checkpoint_cu_starts.copy_( + torch.from_numpy(starts).pin_memory(), non_blocking=True + ) + self.checkpoint_indices.copy_( + torch.from_numpy(indices).pin_memory(), non_blocking=True + ) + for wrapper in self.wrappers.values(): + wrapper.plan(self.cu_seqlens) + + def run(self, *, layer, q, k, v, g, beta, ssm_states, h_track): + num_tokens = q.shape[1] + if num_tokens <= self.cache_indices.numel(): + # FlashInfer selects prefill only when T > number of sequence slots. + # Small buckets have one slot per token, including empty sequences. + # An unused trailing token keeps the prefill dispatch; cu_seqlens + # excludes it, so it cannot read or update any request state. + q, k, v, g, beta = ( + F.pad(x, (0, 0) * (x.ndim - 2) + (0, 1)) for x in (q, k, v, g, beta) + ) + wrapper = self.wrappers.get(layer.layer_id) + if wrapper is None: + wrapper = self.wrapper_cls(q.device) + wrapper.plan(self.cu_seqlens) + self.wrappers[layer.layer_id] = wrapper + checkpoints = {} + if self.track: + checkpoints = dict( + state_checkpoints=h_track, + checkpoint_cu_starts=self.checkpoint_cu_starts, + checkpoint_state_indices=self.checkpoint_indices, + checkpoint_every_n_tokens=64, + ) + return wrapper.run( + q=q.contiguous(), + k=k.contiguous(), + v=v.contiguous(), + g=g.contiguous(), + beta=beta.contiguous(), + A_log=layer.A_log.reshape(-1).float().contiguous(), + dt_bias=layer.dt_bias.reshape(-1).float().contiguous(), + initial_state=ssm_states, + ssm_state_indices=self.cache_indices, + output_final_state=False, + use_qk_l2norm_in_kernel=True, + use_gate_in_kernel=True, + lower_bound=layer.lower_bound, + beta_is_logit=True, + output=torch.empty_like(q), + **checkpoints, + )[0][:, :num_tokens] diff --git a/python/sglang/srt/layers/attention/linear/kda_prefill_graph.py b/python/sglang/srt/layers/attention/linear/kda_prefill_graph.py index f1eb37360b..fb98dc70f4 100644 --- a/python/sglang/srt/layers/attention/linear/kda_prefill_graph.py +++ b/python/sglang/srt/layers/attention/linear/kda_prefill_graph.py @@ -44,6 +44,9 @@ from sglang.kernels.ops.mamba.kda_prefill_track import ( kda_track_conv_window, kda_track_ssm_state, ) +from sglang.srt.layers.attention.linear.kda_flashinfer_prefill_graph import ( + FlashInferKDAPrefillGraph, +) from sglang.srt.layers.attention.mamba.prefill_track_metadata import ( build_prefill_track_plan, ) @@ -101,6 +104,7 @@ class KDAPrefillGraphMetadata(msgspec.Struct, frozen=True): has_initial_state: torch.Tensor # [max_seqs] bool track_dst: torch.Tensor # [max_seqs] int64 token_index: torch.Tensor # [num_tokens] int32 arange + flashinfer: Optional[FlashInferKDAPrefillGraph] = None @classmethod def allocate( @@ -111,6 +115,7 @@ class KDAPrefillGraphMetadata(msgspec.Struct, frozen=True): track: bool, cache_index_dtype: torch.dtype, device: torch.device, + flashinfer_wrapper_cls=None, ) -> KDAPrefillGraphMetadata: max_seqs = max(1, min(num_tokens, max_seqs_cap)) max_chunks = triton.cdiv(num_tokens, KDA_CHUNK_SIZE) + max_seqs @@ -149,6 +154,17 @@ class KDAPrefillGraphMetadata(msgspec.Struct, frozen=True): has_initial_state=torch.zeros(max_seqs, dtype=torch.bool, device=device), track_dst=torch.zeros(max_seqs, dtype=torch.int64, device=device), token_index=torch.arange(num_tokens, dtype=torch.int32, device=device), + flashinfer=( + FlashInferKDAPrefillGraph( + wrapper_cls=flashinfer_wrapper_cls, + num_tokens=num_tokens, + max_seqs=max_seqs, + track=track, + device=device, + ) + if flashinfer_wrapper_cls is not None + else None + ), ) def can_fit(self, extend_seq_lens: list[int]) -> bool: @@ -185,6 +201,8 @@ class KDAPrefillGraphMetadata(msgspec.Struct, frozen=True): self.has_initial_state[bs:].fill_(False) if track_rows is not None: self.track_dst[:bs].copy_(track_rows.dst[:bs]) + if self.flashinfer is not None: + self.flashinfer.fill(self, extend_seq_lens, track_rows) def _write_host_tables( self, *, lens: np.ndarray, track_rows: Optional[KDAPrefillTrackRows] @@ -350,27 +368,39 @@ def kda_extend_in_graph( dtype=torch.float32, device=ssm_states.device, ) - core_attn_out = kernel_dispatcher.extend( - q=q, - k=k, - v=v, - g=a, - beta=b, - ssm_states=ssm_states, - cache_indices=meta.cache_indices, - query_start_loc=meta.cu_seqlens, - A_log=layer.A_log, - dt_bias=layer.dt_bias, - lower_bound=layer.lower_bound, - beta_is_raw=gate_was_flat, - extend_seq_lens_cpu=None, - is_spec_decode=False, - return_intermediate_states=False, - track_state=h_track, - track_chunk_idx=meta.track_chunk_idx if meta.track else None, - chunk_indices=meta.chunk_indices, - chunk_offsets=meta.chunk_offsets, - ) + if meta.flashinfer is not None: + core_attn_out = meta.flashinfer.run( + layer=layer, + q=q, + k=k, + v=v, + g=a, + beta=b, + ssm_states=ssm_states, + h_track=h_track, + ) + else: + core_attn_out = kernel_dispatcher.extend( + q=q, + k=k, + v=v, + g=a, + beta=b, + ssm_states=ssm_states, + cache_indices=meta.cache_indices, + query_start_loc=meta.cu_seqlens, + A_log=layer.A_log, + dt_bias=layer.dt_bias, + lower_bound=layer.lower_bound, + beta_is_raw=gate_was_flat, + extend_seq_lens_cpu=None, + is_spec_decode=False, + return_intermediate_states=False, + track_state=h_track, + track_chunk_idx=meta.track_chunk_idx if meta.track else None, + chunk_indices=meta.chunk_indices, + chunk_offsets=meta.chunk_offsets, + ) if meta.track: kda_track_ssm_state( diff --git a/python/sglang/srt/layers/attention/linear/kernels/kda_flashinfer_prefill.py b/python/sglang/srt/layers/attention/linear/kernels/kda_flashinfer_prefill.py index a09272d9f9..d7ca998c3b 100644 --- a/python/sglang/srt/layers/attention/linear/kernels/kda_flashinfer_prefill.py +++ b/python/sglang/srt/layers/attention/linear/kernels/kda_flashinfer_prefill.py @@ -77,9 +77,13 @@ class FlashInferKDAPrefillKernel(LinearAttnKernelBase): def __init__(self): if torch.cuda.get_device_capability() not in ((10, 0), (10, 3)): raise ValueError("FlashInfer KDA prefill requires SM100 or SM103") - from flashinfer.kda import recurrent_kda + from flashinfer import kda - self._recurrent_kda = recurrent_kda + self._recurrent_kda = kda.recurrent_kda + # Older FlashInfer installations retain the eager BCG break. + self.prefill_graph_wrapper_cls = getattr( + kda, "RecurrentKDAPrefillWrapper", None + ) def decode(self, *args, **kwargs): raise NotImplementedError("FlashInferKDAPrefillKernel is prefill-only") diff --git a/test/registered/kernels/ops/attention/test_kda_prefill_graph.py b/test/registered/kernels/ops/attention/test_kda_prefill_graph.py index 2e68938587..69edb88384 100644 --- a/test/registered/kernels/ops/attention/test_kda_prefill_graph.py +++ b/test/registered/kernels/ops/attention/test_kda_prefill_graph.py @@ -4,8 +4,10 @@ eager KDAAttnBackend.forward_extend. One bucket is captured once, then replayed for batches of different shapes: several sequences, cached prefixes, padding past the batch, chunk-aligned and unaligned prefix-cache snapshots, and a batch without tracking. Each replay must -match the eager path bit for bit on the output, the conv / SSM state pools and -the track slots, and must leave every other pool slot untouched. +match the eager path on the output, the conv / SSM state pools and the track +slots, and must leave every other pool slot untouched. Comparisons are bitwise +except FlashInfer's small padded plans and the eager Triton fallback for +one-token sequences, which can use different floating-point operation orders. """ import unittest @@ -58,7 +60,7 @@ def _layer(): ) -def _backend(conv_pool, ssm_pool): +def _backend(conv_pool, ssm_pool, prefill_backend=LinearAttnKernelBackend.TRITON): """The production backend on test pools: slot of request i is i + 1.""" backend = KDAAttnBackend.__new__(KDAAttnBackend) cache = SimpleNamespace(conv=[conv_pool], temporal=ssm_pool) @@ -76,7 +78,7 @@ def _backend(conv_pool, ssm_pool): backend.prefill_graph_metadata = None backend.kernel_dispatcher = KDAKernelDispatcher( LinearAttnKernelBackend.TRITON, - LinearAttnKernelBackend.TRITON, + prefill_backend, LinearAttnKernelBackend.TRITON, ) return backend @@ -127,6 +129,8 @@ def _batch(extend_lens, prefix_lens, track=None): class TestKDAPrefillGraph(CustomTestCase): + prefill_backend = LinearAttnKernelBackend.TRITON + def setUp(self): self.layer = _layer() g = torch.Generator(device="cuda").manual_seed(11) @@ -139,7 +143,7 @@ class TestKDAPrefillGraph(CustomTestCase): ) * 0.05 ) - self.backend = _backend(self.conv_pool, self.ssm_pool) + self.backend = _backend(self.conv_pool, self.ssm_pool, self.prefill_backend) # Static inputs the captured graph reads. self.mixed = torch.zeros(BUCKET, DIM, device="cuda", dtype=torch.bfloat16) self.a = torch.zeros(1, BUCKET, HEADS * HEAD_DIM, device="cuda") @@ -157,21 +161,26 @@ class TestKDAPrefillGraph(CustomTestCase): cache_indices=self.backend._prefill_graph_cache_indices(batch), ) - def _capture(self): - meta = KDAPrefillGraphMetadata.allocate( - num_tokens=BUCKET, - max_seqs_cap=MAX_SEQS, - track=True, - cache_index_dtype=torch.int32, - device=torch.device("cuda"), - ) - capture_batch = _batch([BUCKET], [0]) - self._fill(meta, capture_batch) + def _capture(self, *, track=True, num_tokens=BUCKET, max_seqs=MAX_SEQS): + from sglang.srt.runtime_context import get_context + + capture_batch = _batch([num_tokens], [0]) + capture_batch.extend_num_tokens = num_tokens + capture_batch.positions = torch.arange(num_tokens, device="cuda") + self.backend.prefill_graph_max_seqs = max_seqs + with get_context().override_server_args( + disable_radix_cache=not track, mamba_radix_cache_strategy="extra_buffer" + ): + meta = self.backend.init_prefill_graph_metadata(capture_batch) snapshot = (self.conv_pool.clone(), self.ssm_pool.clone()) def run(): return self.backend.forward_extend( - self.layer, capture_batch, self.mixed, self.a, self.b + self.layer, + capture_batch, + self.mixed[:num_tokens], + self.a[:, :num_tokens], + self.b[:, :num_tokens], ) stream = torch.cuda.Stream() @@ -180,7 +189,7 @@ class TestKDAPrefillGraph(CustomTestCase): run() # warmup: Triton compile / autotune outside the capture torch.cuda.current_stream().wait_stream(stream) graph = torch.cuda.CUDAGraph() - with torch.cuda.graph(graph): + with torch.cuda.graph(graph, stream=stream): out = run() # Warmup and capture ran on the pools; restore them. self.conv_pool.copy_(snapshot[0]) @@ -205,7 +214,7 @@ class TestKDAPrefillGraph(CustomTestCase): # Eager reference on cloned pools. ref_conv, ref_ssm = self.conv_pool.clone(), self.ssm_pool.clone() - ref_backend = _backend(ref_conv, ref_ssm) + ref_backend = _backend(ref_conv, ref_ssm, self.prefill_backend) ref_batch = _batch(lens, prefixes, track=track) ref_backend.init_forward_metadata(ref_batch) ref_out = ref_backend.forward_extend( @@ -220,10 +229,34 @@ class TestKDAPrefillGraph(CustomTestCase): graph.replay() torch.cuda.synchronize() - self.assertTrue(torch.equal(out[:, :total], ref_out[:, :total])) + different_kernel_schedule = ( + self.prefill_backend == LinearAttnKernelBackend.FLASHINFER + and (max(lens) == 1 or meta.num_tokens <= meta.max_seqs) + ) + if different_kernel_schedule: + # Planned small FlashInfer buckets and the eager Triton fallback + # need numerical comparison. Unused slots must remain bitwise equal. + torch.testing.assert_close( + out[:, :total], ref_out[:, :total], rtol=1e-2, atol=1e-4 + ) + touched = list(range(1, len(lens) + 1)) + touched += [NUM_SLOTS // 2 + row for row in (track or {})] + untouched = [slot for slot in range(NUM_SLOTS) if slot not in touched] + torch.testing.assert_close( + self.ssm_pool[touched], ref_ssm[touched], rtol=1e-2, atol=1e-3 + ) + self.assertTrue(torch.equal(self.ssm_pool[untouched], ref_ssm[untouched])) + else: + self.assertTrue(torch.equal(out[:, :total], ref_out[:, :total])) + self.assertTrue(torch.equal(self.ssm_pool, ref_ssm), "ssm pool differs") self.assertTrue(bool((out[:, total:] == 0).all()), "padded rows not zeroed") self.assertTrue(torch.equal(self.conv_pool, ref_conv), "conv pool differs") - self.assertTrue(torch.equal(self.ssm_pool, ref_ssm), "ssm pool differs") + + def test_without_radix_tracking(self): + meta, graph, out = self._capture(track=False) + self._check_replay( + meta, graph, out, lens=[129, 7], prefixes=[64, 0], track=None, seed=300 + ) def test_replays_match_eager(self): meta, graph, out = self._capture() @@ -240,6 +273,7 @@ class TestKDAPrefillGraph(CustomTestCase): ), ([200, 11], [0, 0], None), ([1], [900], None), + ([1] * MAX_SEQS, [64] * MAX_SEQS, None), ] for i, (lens, prefixes, track) in enumerate(cases): with self.subTest(case=i): @@ -270,5 +304,33 @@ class TestKDAPrefillGraph(CustomTestCase): self.assertLessEqual(chunks, meta.max_chunks) +class TestFlashInferKDAPrefillGraph(TestKDAPrefillGraph): + prefill_backend = LinearAttnKernelBackend.FLASHINFER + + def test_small_buckets_with_padded_sequences(self): + for num_tokens in (1, 8, 16, 128): + with self.subTest(num_tokens=num_tokens): + meta, graph, out = self._capture(num_tokens=num_tokens, max_seqs=128) + for lens in ([num_tokens], [1] * min(num_tokens, MAX_SEQS)): + self._check_replay( + meta, + graph, + out, + lens=lens, + prefixes=[64] * len(lens), + track={0: 70} if lens == [128] else None, + seed=400 + num_tokens, + ) + + @classmethod + def setUpClass(cls): + if torch.cuda.get_device_capability() not in ((10, 0), (10, 3)): + raise unittest.SkipTest("FlashInfer KDA prefill requires SM100/SM103") + from flashinfer import kda + + if not hasattr(kda, "RecurrentKDAPrefillWrapper"): + raise unittest.SkipTest("FlashInfer lacks the KDA prefill graph wrapper") + + if __name__ == "__main__": unittest.main()