[Perf] Compact GDN causal-conv grids for skewed prefill batches (#43570)

This commit is contained in:
Xiaoyu Zhang
2026-10-13 17:45:06 +08:00
committed by GitHub
parent 01f6840693
commit 4e0fa52530
5 changed files with 170 additions and 3 deletions
@@ -5,6 +5,7 @@ import torch
from sglang.kernels.ops.attention.fla.fused_gdn_gating import fused_gdn_gating
from sglang.kernels.ops.mamba.causal_conv1d_triton import (
CAUSAL_CONV1D_FWD_BLOCK_M,
causal_conv1d_fn,
causal_conv1d_update,
)
@@ -21,11 +22,11 @@ from sglang.srt.layers.attention.linear.utils import (
)
from sglang.srt.layers.radix_linear_attention import RadixLinearAttention
from sglang.srt.mem_cache.memory_pool import MambaPool
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
from sglang.srt.model_executor.model_runner import ModelRunner
from sglang.srt.runtime_context import get_exec, get_memory, get_schedule
from sglang.srt.utils import is_cpu, is_cuda, is_hip, is_npu, is_xpu
from sglang.srt.utils.common import rank0_log
from sglang.srt.utils.common import is_pin_memory_available, rank0_log
_is_hip = is_hip()
@@ -581,6 +582,44 @@ class GDNAttnBackend(MambaAttnBackendBase):
maybe_build_flashinfer_checkpoint_plan(
forward_batch, self.forward_metadata, self.device
)
self._init_prefill_conv_block_table(forward_batch)
def _init_prefill_conv_block_table(self, forward_batch: ForwardBatch) -> None:
metadata = self.forward_metadata
metadata.conv_block_table = None
if (
not is_cuda()
or forward_batch.forward_mode not in (ForwardMode.EXTEND, ForwardMode.MIXED)
or self.mis_metadata is not None
or forward_batch.tbo_parent_token_range is not None
):
return
seq_lens = forward_batch.extend_seq_lens_cpu
if seq_lens is None or len(seq_lens) != metadata.query_start_loc.numel() - 1:
return
block_counts = [
(length + CAUSAL_CONV1D_FWD_BLOCK_M - 1) // CAUSAL_CONV1D_FWD_BLOCK_M
for length in seq_lens
]
# Amortize table construction only when most of the rectangular grid
# would be empty: a 4x ratio means over 75% of its blocks do no work.
# This conservative crossover avoids charging mildly ragged batches
# for the host table and H2D copy, measured at ~280 us per forward.
if not block_counts or len(block_counts) * max(block_counts) <= 4 * sum(
block_counts
):
return
# Build once per forward for all GDN layers. A rectangular launch on a
# mixed batch otherwise schedules every decode row for the longest prefill.
pairs = [
(seq, block)
for seq, count in enumerate(block_counts)
for block in range(count)
]
metadata.conv_block_table = torch.tensor(
pairs, dtype=torch.int32, device="cpu", pin_memory=is_pin_memory_available()
).to(self.device, non_blocking=True)
def _init_target_verify_qkv_routing(self, forward_batch: ForwardBatch) -> None:
# CUDA-graph metadata leaves mode and draft count at their defaults.
@@ -886,6 +925,8 @@ class GDNAttnBackend(MambaAttnBackendBase):
conv_states: torch.Tensor,
cache_indices: torch.Tensor,
) -> torch.Tensor:
block_table = self.forward_metadata.conv_block_table
conv_kwargs = {"block_table": block_table} if block_table is not None else {}
return causal_conv1d_fn(
mixed_qkv.transpose(0, 1),
layer.conv_weights,
@@ -896,6 +937,7 @@ class GDNAttnBackend(MambaAttnBackendBase):
cache_indices=cache_indices,
query_start_loc=self.forward_metadata.query_start_loc,
seq_lens_cpu=forward_batch.extend_seq_lens_cpu,
**conv_kwargs,
).transpose(0, 1)[: mixed_qkv.shape[0]]
@staticmethod
@@ -29,6 +29,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch
class ForwardMetadata:
query_start_loc: torch.Tensor
mamba_cache_indices: torch.Tensor
conv_block_table: Optional[torch.Tensor] = None
logical_num_tokens: Optional[int] = None
mamba_cache_indices_gdn: Optional[torch.Tensor] = None
# Mamba track DESTINATION slots (PHYSICAL, length == batch). Like