[diffusion] optimization: route subBlock sparse attention in head chunks (#41985)

This commit is contained in:
saatwiknagpal
2026-10-03 15:54:54 +08:00
committed by GitHub
parent 254aa41f78
commit 1093c501df
2 changed files with 44 additions and 0 deletions
@@ -103,6 +103,9 @@ DEFAULT_MIN_SEQ_LEN = 4096
# Keep the established BF16 kernel as the default. ``sage_fp8`` is explicit
# until its model-level quality/performance envelope has been validated.
DEFAULT_COMPUTE_MODE = "bf16"
# Caps the [H, Gq, Gk] routing scores and block index of one call at 2 GiB each;
# larger calls (e.g. every head on one GPU at long sequences) are split by heads.
_MAX_ROUTED_BLOCK_PAIRS = 2**29
# ``blocks.<idx>.attn`` is a DiT layer; ``token_refiner.blocks.<idx>.attn`` and
# anything else is not and stays dense.
@@ -567,6 +570,33 @@ class SubBlockSparseAttentionImpl(AttentionImpl):
sparse_query_block_mask: torch.Tensor | None = None,
) -> torch.Tensor:
"""Q ``[1, Sq, H, 128]`` against K/V ``[1, Sk, H, 128]``."""
block_pairs = -(-q.shape[1] // SUBBLOCK_SPARSE_BLOCK_SIZE) * -(
-k.shape[1] // SUBBLOCK_SPARSE_BLOCK_SIZE
)
heads_per_call = max(1, _MAX_ROUTED_BLOCK_PAIRS // block_pairs)
if q.shape[2] <= heads_per_call:
return self._sparse_attention_heads(
q, k, v, sparse_query_block_mask=sparse_query_block_mask
)
out = torch.empty_like(q)
for start in range(0, q.shape[2], heads_per_call):
heads = slice(start, start + heads_per_call)
out[:, :, heads] = self._sparse_attention_heads(
q[:, :, heads],
k[:, :, heads],
v[:, :, heads],
sparse_query_block_mask=sparse_query_block_mask,
)
return out
def _sparse_attention_heads(
self,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
*,
sparse_query_block_mask: torch.Tensor | None = None,
) -> torch.Tensor:
plan = self.router.route(
q,
k,
@@ -545,6 +545,20 @@ class TestSubBlockNumerics(unittest.TestCase):
self.skipTest("requires the SM120 SubBlock kernel")
self._assert_kernel_backed_mixed_query_mask()
def test_head_chunks_match_one_call(self):
"""Splitting a call by heads must not change its output."""
q, k, v = _structured_qkv(self.seq_len, torch.device("cuda"))
impl = self._impl(sparsity=0.75)
mask = torch.arange(self.seq_len // 64, device="cuda") >= self.seq_len // 128
whole = impl._sparse_attention(q, k, v, sparse_query_block_mask=mask)
one_head = (self.seq_len // 64) ** 2
with patch(
"sglang.multimodal_gen.runtime.layers.attention.backends.subblock_sparse_attn._MAX_ROUTED_BLOCK_PAIRS",
one_head,
):
chunked = impl._sparse_attention(q, k, v, sparse_query_block_mask=mask)
torch.testing.assert_close(chunked, whole, rtol=0, atol=0)
def test_skipped_step_is_bitwise_dense(self):
device = torch.device("cuda")
q, k, v = _structured_qkv(self.seq_len, device)