diff --git a/python/sglang/srt/layers/attention/linear/gdn_backend.py b/python/sglang/srt/layers/attention/linear/gdn_backend.py index 4bf5aaec6e..75782942ff 100644 --- a/python/sglang/srt/layers/attention/linear/gdn_backend.py +++ b/python/sglang/srt/layers/attention/linear/gdn_backend.py @@ -5,6 +5,7 @@ import torch from sglang.kernels.ops.attention.fla.fused_gdn_gating import fused_gdn_gating from sglang.kernels.ops.mamba.causal_conv1d_triton import ( + CAUSAL_CONV1D_FWD_BLOCK_M, causal_conv1d_fn, causal_conv1d_update, ) @@ -21,11 +22,11 @@ from sglang.srt.layers.attention.linear.utils import ( ) from sglang.srt.layers.radix_linear_attention import RadixLinearAttention from sglang.srt.mem_cache.memory_pool import MambaPool -from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode from sglang.srt.model_executor.model_runner import ModelRunner from sglang.srt.runtime_context import get_exec, get_memory, get_schedule from sglang.srt.utils import is_cpu, is_cuda, is_hip, is_npu, is_xpu -from sglang.srt.utils.common import rank0_log +from sglang.srt.utils.common import is_pin_memory_available, rank0_log _is_hip = is_hip() @@ -581,6 +582,44 @@ class GDNAttnBackend(MambaAttnBackendBase): maybe_build_flashinfer_checkpoint_plan( forward_batch, self.forward_metadata, self.device ) + self._init_prefill_conv_block_table(forward_batch) + + def _init_prefill_conv_block_table(self, forward_batch: ForwardBatch) -> None: + metadata = self.forward_metadata + metadata.conv_block_table = None + if ( + not is_cuda() + or forward_batch.forward_mode not in (ForwardMode.EXTEND, ForwardMode.MIXED) + or self.mis_metadata is not None + or forward_batch.tbo_parent_token_range is not None + ): + return + seq_lens = forward_batch.extend_seq_lens_cpu + if seq_lens is None or len(seq_lens) != metadata.query_start_loc.numel() - 1: + return + block_counts = [ + (length + CAUSAL_CONV1D_FWD_BLOCK_M - 1) // CAUSAL_CONV1D_FWD_BLOCK_M + for length in seq_lens + ] + # Amortize table construction only when most of the rectangular grid + # would be empty: a 4x ratio means over 75% of its blocks do no work. + # This conservative crossover avoids charging mildly ragged batches + # for the host table and H2D copy, measured at ~280 us per forward. + if not block_counts or len(block_counts) * max(block_counts) <= 4 * sum( + block_counts + ): + return + + # Build once per forward for all GDN layers. A rectangular launch on a + # mixed batch otherwise schedules every decode row for the longest prefill. + pairs = [ + (seq, block) + for seq, count in enumerate(block_counts) + for block in range(count) + ] + metadata.conv_block_table = torch.tensor( + pairs, dtype=torch.int32, device="cpu", pin_memory=is_pin_memory_available() + ).to(self.device, non_blocking=True) def _init_target_verify_qkv_routing(self, forward_batch: ForwardBatch) -> None: # CUDA-graph metadata leaves mode and draft count at their defaults. @@ -886,6 +925,8 @@ class GDNAttnBackend(MambaAttnBackendBase): conv_states: torch.Tensor, cache_indices: torch.Tensor, ) -> torch.Tensor: + block_table = self.forward_metadata.conv_block_table + conv_kwargs = {"block_table": block_table} if block_table is not None else {} return causal_conv1d_fn( mixed_qkv.transpose(0, 1), layer.conv_weights, @@ -896,6 +937,7 @@ class GDNAttnBackend(MambaAttnBackendBase): cache_indices=cache_indices, query_start_loc=self.forward_metadata.query_start_loc, seq_lens_cpu=forward_batch.extend_seq_lens_cpu, + **conv_kwargs, ).transpose(0, 1)[: mixed_qkv.shape[0]] @staticmethod diff --git a/python/sglang/srt/layers/attention/mamba/mamba2_metadata.py b/python/sglang/srt/layers/attention/mamba/mamba2_metadata.py index cea43f7ebf..41eded1125 100644 --- a/python/sglang/srt/layers/attention/mamba/mamba2_metadata.py +++ b/python/sglang/srt/layers/attention/mamba/mamba2_metadata.py @@ -29,6 +29,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch class ForwardMetadata: query_start_loc: torch.Tensor mamba_cache_indices: torch.Tensor + conv_block_table: Optional[torch.Tensor] = None logical_num_tokens: Optional[int] = None mamba_cache_indices_gdn: Optional[torch.Tensor] = None # Mamba track DESTINATION slots (PHYSICAL, length == batch). Like diff --git a/test/registered/attention/test_gdn_prefill_layout.py b/test/registered/attention/test_gdn_prefill_layout.py index 497004b4ef..c40ddd945f 100644 --- a/test/registered/attention/test_gdn_prefill_layout.py +++ b/test/registered/attention/test_gdn_prefill_layout.py @@ -1,4 +1,5 @@ import unittest +from types import SimpleNamespace import torch @@ -11,6 +12,9 @@ from sglang.kernels.ops.attention.triton_gdn_fused_proj import ( fused_qkv_split_gdn_prefill, qwen3_5_gdn_prefill_projection_views, ) +from sglang.srt.layers.attention.linear.gdn_backend import GDNAttnBackend +from sglang.srt.layers.attention.mamba.mamba2_metadata import ForwardMetadata +from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.test.ci.ci_register import register_cuda_ci register_cuda_ci(est_time=6, stage="base-b", runner_config="1-gpu-large") @@ -23,6 +27,62 @@ class TestGdnPrefillLayout(unittest.TestCase): NUM_V_HEADS = 4 HEAD_DIM = 128 + def test_compact_prefill_conv_matches_rectangular_output_and_cache(self): + torch.manual_seed(0) + for seq_lens in ([65, 1, 0, 1, 1, 1, 1, 1], [8065] + [1] * 127): + with self.subTest(batch_size=len(seq_lens)): + dim, width = 768, 4 + batch_size = len(seq_lens) + lengths = torch.tensor(seq_lens, device="cuda", dtype=torch.int32) + starts = torch.cat([lengths.new_zeros(1), lengths.cumsum(0)]).int() + # Preserve the token-major projection view, including the unused Z. + qkv = torch.randn( + sum(seq_lens), dim + 256, device="cuda", dtype=torch.bfloat16 + )[:, :dim] + cache_indices = torch.randperm(batch_size + 2, device="cuda")[ + :batch_size + ] + cache_indices = cache_indices.int() + cache_indices[-1] = -1 + states = torch.randn( + batch_size + 2, dim, width - 1, device="cuda", dtype=qkv.dtype + ) + expected_states = states.clone() + layer = SimpleNamespace( + conv_weights=torch.randn( + dim, width, device="cuda", dtype=qkv.dtype + ), + bias=torch.randn(dim, device="cuda", dtype=qkv.dtype), + activation="silu", + ) + batch = SimpleNamespace( + forward_mode=ForwardMode.MIXED, + tbo_parent_token_range=None, + extend_seq_lens_cpu=seq_lens, + extend_prefix_lens=torch.ones(batch_size, device="cuda"), + ) + batch.extend_prefix_lens[0] = 0 + backend = object.__new__(GDNAttnBackend) + backend.device = "cuda" + backend.mis_metadata = None + backend.forward_metadata = ForwardMetadata( + query_start_loc=starts, mamba_cache_indices=cache_indices + ) + expected = backend._convolve_prefill( + layer, batch, qkv, expected_states, cache_indices + ) + backend._init_prefill_conv_block_table(batch) + self.assertIsNotNone(backend.forward_metadata.conv_block_table) + actual = backend._convolve_prefill( + layer, batch, qkv, states, cache_indices + ) + # Padded rows have no defined output; all cache slots must still match. + valid_tokens = sum(seq_lens[:-1]) + torch.testing.assert_close( + actual[:valid_tokens], expected[:valid_tokens], rtol=0, atol=0 + ) + torch.testing.assert_close(states, expected_states, rtol=0, atol=0) + def _projection_views(self, dtype): qkv_dim = ( 2 * self.NUM_QK_HEADS * self.HEAD_DIM + self.NUM_V_HEADS * self.HEAD_DIM diff --git a/test/registered/unit/layers/attention/test_gdn_prefill_backend_policy.py b/test/registered/unit/layers/attention/test_gdn_prefill_backend_policy.py index 182ee17a86..30cd0402e8 100644 --- a/test/registered/unit/layers/attention/test_gdn_prefill_backend_policy.py +++ b/test/registered/unit/layers/attention/test_gdn_prefill_backend_policy.py @@ -21,6 +21,8 @@ from sglang.srt.layers.attention.linear.utils import ( LinearAttnKernelBackend, resolve_linear_attn_backends, ) +from sglang.srt.layers.attention.mamba.mamba2_metadata import ForwardMetadata +from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.runtime_context import get_context from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -79,6 +81,66 @@ def make_runner( class TestFlashInferGDNPrefillBackendPolicy(CustomTestCase): + def test_prefill_conv_table_covers_only_nonempty_chunks(self): + backend = object.__new__(GDNAttnBackend) + backend.device = "cpu" + backend.mis_metadata = None + backend.forward_metadata = ForwardMetadata( + query_start_loc=torch.tensor([0, 65, 66, 66, 67, 68, 69, 70, 71]), + mamba_cache_indices=torch.arange(8), + ) + batch = SimpleNamespace( + forward_mode=ForwardMode.MIXED, + tbo_parent_token_range=None, + extend_seq_lens_cpu=[65, 1, 0, 1, 1, 1, 1, 1], + ) + with ( + patch.object(gdn_backend, "is_cuda", return_value=True), + patch.object(gdn_backend, "is_pin_memory_available", return_value=False), + ): + backend._init_prefill_conv_block_table(batch) + torch.testing.assert_close( + backend.forward_metadata.conv_block_table, + torch.tensor( + [[0, i] for i in range(9)] + [[i, 0] for i in (1, 3, 4, 5, 6, 7)], + dtype=torch.int32, + ), + ) + + def test_prefill_conv_table_falls_back_and_clears_previous_batch(self): + cases = ( + ("uniform", {"extend_seq_lens_cpu": [7, 8]}, {}), + ("mildly_ragged", {}, {}), + ("no_cpu_lengths", {"extend_seq_lens_cpu": None}, {}), + ("different_batch", {"extend_seq_lens_cpu": [17, 1, 1]}, {}), + ("tbo", {"tbo_parent_token_range": (0, 18)}, {}), + ("decode", {"forward_mode": ForwardMode.DECODE}, {}), + ("verify", {"forward_mode": ForwardMode.TARGET_VERIFY}, {}), + ("multi_item_scoring", {}, {"mis_metadata": sentinel.metadata}), + ("non_cuda", {}, {"cuda": False}), + ) + for name, batch_fields, options in cases: + with self.subTest(name=name): + backend = object.__new__(GDNAttnBackend) + backend.device = "cpu" + backend.mis_metadata = options.get("mis_metadata") + backend.forward_metadata = ForwardMetadata( + query_start_loc=torch.tensor([0, 17, 18]), + mamba_cache_indices=torch.arange(2), + conv_block_table=sentinel.previous_batch, + ) + fields = dict( + forward_mode=ForwardMode.EXTEND, + tbo_parent_token_range=None, + extend_seq_lens_cpu=[17, 1], + ) + fields.update(batch_fields) + with patch.object( + gdn_backend, "is_cuda", return_value=options.get("cuda", True) + ): + backend._init_prefill_conv_block_table(SimpleNamespace(**fields)) + self.assertIsNone(backend.forward_metadata.conv_block_table) + @staticmethod def make_target_verify_routing_backend(): backend = object.__new__(GDNAttnBackend) diff --git a/test/registered/unit/layers/attention/test_mamba_track_state_dtype.py b/test/registered/unit/layers/attention/test_mamba_track_state_dtype.py index 6b75c40903..7560004fa5 100644 --- a/test/registered/unit/layers/attention/test_mamba_track_state_dtype.py +++ b/test/registered/unit/layers/attention/test_mamba_track_state_dtype.py @@ -113,7 +113,9 @@ class TestTrackMambaStateDtype(CustomTestCase): with self.subTest(backend=backend_type.__name__): backend = object.__new__(backend_type) starts = torch.tensor([0, 2, 4]) - backend.forward_metadata = SimpleNamespace(query_start_loc=starts) + backend.forward_metadata = SimpleNamespace( + query_start_loc=starts, conv_block_table=None + ) with patch.object( module, "causal_conv1d_fn", return_value=qkv.T ) as conv: