[diffusion] optimization: route subBlock sparse attention in head chunks (#41985)
This commit is contained in:
+30
@@ -103,6 +103,9 @@ DEFAULT_MIN_SEQ_LEN = 4096
|
||||
# Keep the established BF16 kernel as the default. ``sage_fp8`` is explicit
|
||||
# until its model-level quality/performance envelope has been validated.
|
||||
DEFAULT_COMPUTE_MODE = "bf16"
|
||||
# Caps the [H, Gq, Gk] routing scores and block index of one call at 2 GiB each;
|
||||
# larger calls (e.g. every head on one GPU at long sequences) are split by heads.
|
||||
_MAX_ROUTED_BLOCK_PAIRS = 2**29
|
||||
|
||||
# ``blocks.<idx>.attn`` is a DiT layer; ``token_refiner.blocks.<idx>.attn`` and
|
||||
# anything else is not and stays dense.
|
||||
@@ -567,6 +570,33 @@ class SubBlockSparseAttentionImpl(AttentionImpl):
|
||||
sparse_query_block_mask: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Q ``[1, Sq, H, 128]`` against K/V ``[1, Sk, H, 128]``."""
|
||||
block_pairs = -(-q.shape[1] // SUBBLOCK_SPARSE_BLOCK_SIZE) * -(
|
||||
-k.shape[1] // SUBBLOCK_SPARSE_BLOCK_SIZE
|
||||
)
|
||||
heads_per_call = max(1, _MAX_ROUTED_BLOCK_PAIRS // block_pairs)
|
||||
if q.shape[2] <= heads_per_call:
|
||||
return self._sparse_attention_heads(
|
||||
q, k, v, sparse_query_block_mask=sparse_query_block_mask
|
||||
)
|
||||
out = torch.empty_like(q)
|
||||
for start in range(0, q.shape[2], heads_per_call):
|
||||
heads = slice(start, start + heads_per_call)
|
||||
out[:, :, heads] = self._sparse_attention_heads(
|
||||
q[:, :, heads],
|
||||
k[:, :, heads],
|
||||
v[:, :, heads],
|
||||
sparse_query_block_mask=sparse_query_block_mask,
|
||||
)
|
||||
return out
|
||||
|
||||
def _sparse_attention_heads(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
*,
|
||||
sparse_query_block_mask: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
plan = self.router.route(
|
||||
q,
|
||||
k,
|
||||
|
||||
@@ -545,6 +545,20 @@ class TestSubBlockNumerics(unittest.TestCase):
|
||||
self.skipTest("requires the SM120 SubBlock kernel")
|
||||
self._assert_kernel_backed_mixed_query_mask()
|
||||
|
||||
def test_head_chunks_match_one_call(self):
|
||||
"""Splitting a call by heads must not change its output."""
|
||||
q, k, v = _structured_qkv(self.seq_len, torch.device("cuda"))
|
||||
impl = self._impl(sparsity=0.75)
|
||||
mask = torch.arange(self.seq_len // 64, device="cuda") >= self.seq_len // 128
|
||||
whole = impl._sparse_attention(q, k, v, sparse_query_block_mask=mask)
|
||||
one_head = (self.seq_len // 64) ** 2
|
||||
with patch(
|
||||
"sglang.multimodal_gen.runtime.layers.attention.backends.subblock_sparse_attn._MAX_ROUTED_BLOCK_PAIRS",
|
||||
one_head,
|
||||
):
|
||||
chunked = impl._sparse_attention(q, k, v, sparse_query_block_mask=mask)
|
||||
torch.testing.assert_close(chunked, whole, rtol=0, atol=0)
|
||||
|
||||
def test_skipped_step_is_bitwise_dense(self):
|
||||
device = torch.device("cuda")
|
||||
q, k, v = _structured_qkv(self.seq_len, device)
|
||||
|
||||
Reference in New Issue
Block a user