diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/subblock_sparse_attn.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/subblock_sparse_attn.py index 0fa30f9182..e441fdd1c1 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/subblock_sparse_attn.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/subblock_sparse_attn.py @@ -103,6 +103,9 @@ DEFAULT_MIN_SEQ_LEN = 4096 # Keep the established BF16 kernel as the default. ``sage_fp8`` is explicit # until its model-level quality/performance envelope has been validated. DEFAULT_COMPUTE_MODE = "bf16" +# Caps the [H, Gq, Gk] routing scores and block index of one call at 2 GiB each; +# larger calls (e.g. every head on one GPU at long sequences) are split by heads. +_MAX_ROUTED_BLOCK_PAIRS = 2**29 # ``blocks..attn`` is a DiT layer; ``token_refiner.blocks..attn`` and # anything else is not and stays dense. @@ -567,6 +570,33 @@ class SubBlockSparseAttentionImpl(AttentionImpl): sparse_query_block_mask: torch.Tensor | None = None, ) -> torch.Tensor: """Q ``[1, Sq, H, 128]`` against K/V ``[1, Sk, H, 128]``.""" + block_pairs = -(-q.shape[1] // SUBBLOCK_SPARSE_BLOCK_SIZE) * -( + -k.shape[1] // SUBBLOCK_SPARSE_BLOCK_SIZE + ) + heads_per_call = max(1, _MAX_ROUTED_BLOCK_PAIRS // block_pairs) + if q.shape[2] <= heads_per_call: + return self._sparse_attention_heads( + q, k, v, sparse_query_block_mask=sparse_query_block_mask + ) + out = torch.empty_like(q) + for start in range(0, q.shape[2], heads_per_call): + heads = slice(start, start + heads_per_call) + out[:, :, heads] = self._sparse_attention_heads( + q[:, :, heads], + k[:, :, heads], + v[:, :, heads], + sparse_query_block_mask=sparse_query_block_mask, + ) + return out + + def _sparse_attention_heads( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + *, + sparse_query_block_mask: torch.Tensor | None = None, + ) -> torch.Tensor: plan = self.router.route( q, k, diff --git a/python/sglang/multimodal_gen/test/unit/test_subblock_sparse_attention.py b/python/sglang/multimodal_gen/test/unit/test_subblock_sparse_attention.py index 5712fa6b2f..0f59ad4f4d 100644 --- a/python/sglang/multimodal_gen/test/unit/test_subblock_sparse_attention.py +++ b/python/sglang/multimodal_gen/test/unit/test_subblock_sparse_attention.py @@ -545,6 +545,20 @@ class TestSubBlockNumerics(unittest.TestCase): self.skipTest("requires the SM120 SubBlock kernel") self._assert_kernel_backed_mixed_query_mask() + def test_head_chunks_match_one_call(self): + """Splitting a call by heads must not change its output.""" + q, k, v = _structured_qkv(self.seq_len, torch.device("cuda")) + impl = self._impl(sparsity=0.75) + mask = torch.arange(self.seq_len // 64, device="cuda") >= self.seq_len // 128 + whole = impl._sparse_attention(q, k, v, sparse_query_block_mask=mask) + one_head = (self.seq_len // 64) ** 2 + with patch( + "sglang.multimodal_gen.runtime.layers.attention.backends.subblock_sparse_attn._MAX_ROUTED_BLOCK_PAIRS", + one_head, + ): + chunked = impl._sparse_attention(q, k, v, sparse_query_block_mask=mask) + torch.testing.assert_close(chunked, whole, rtol=0, atol=0) + def test_skipped_step_is_bitwise_dense(self): device = torch.device("cuda") q, k, v = _structured_qkv(self.seq_len, device)