From 465ddf9758878a678aecb65f713655afe5ede518 Mon Sep 17 00:00:00 2001 From: Woody-Hu Date: Sat, 17 Oct 2026 18:47:19 +0800 Subject: [PATCH] [Feat] diffusion/VSA-H3: support sm80 (A100) with a raw-pointer Triton load path (#43823) Co-authored-by: Mick --- .../attention/backends/vsa_h3_kernels.py | 147 +++++++++++++++++- .../multimodal_gen/runtime/platforms/cuda.py | 13 +- .../test/unit/test_server_args.py | 8 +- .../test/unit/test_vsa_h3_attention.py | 39 +++++ 4 files changed, 194 insertions(+), 13 deletions(-) diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/vsa_h3_kernels.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/vsa_h3_kernels.py index ab7c44dff4..ad0dee669f 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/vsa_h3_kernels.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/vsa_h3_kernels.py @@ -12,6 +12,12 @@ The attention kernel consumes an explicit per-query-block index list chunks and 3D video tiles whose dimensions do not divide the tile shape - mask their pad columns exactly. +Two kernels implement the same attention with different load paths: a TMA +tensor-descriptor kernel for Hopper and newer, and a raw-pointer kernel for +sm80, where Triton does not lower descriptor loads to TMA. Both compute +identical results; ``vsa_h3_block_sparse_attn_forward`` picks between them +from the device compute capability. + ``vsa_h3_pack_tiles`` gathers packed ``[T, H, D]`` rows into the head-major padded tile layout the attention kernel reads and pools each tile in the same pass; ``vsa_h3_untile`` scatters the attention output back to packed rows and @@ -27,8 +33,6 @@ import triton import triton.language as tl from triton.tools.tensor_descriptor import TensorDescriptor -from sglang.kernels.ops.diffusion import vsa_block_sparse_sm100 - # BLOCK_M / BLOCK_N are structural, not tunable: the kernel indexes the top-k # list per BLOCK_M q-tile and addresses keys as kv_idx * BLOCK_N, so both must # match the granularity q2k_index and variable_block_sizes were built at. @@ -36,9 +40,23 @@ VSA_H3_KERNEL_BLOCK = 64 # Pinned instead of autotuned: on B300 num_warps=4 wins at every sequence # length and num_stages=5 is within 0.5% of the best (7 spills); FastVideo -# reports the same optimum for Blackwell. +# reports the same optimum for Blackwell. The raw-pointer path used on sm80 +# measures best at num_stages=2 on the production DiT shape (H=56, D=128, +# S=37760, keep=66); its loads are not TMA-prefetched, so the deeper pipeline +# the descriptor path relies on does not help there. _ATTN_NUM_WARPS = 4 _ATTN_NUM_STAGES = 5 +_ATTN_NUM_STAGES_PTR = 2 + + +def _use_pointer_loads(capability: tuple[int, int]) -> bool: + """Whether to use the raw-pointer kernel for the given compute capability. + + ``TensorDescriptor`` loads are lowered to TMA only on Hopper and newer. On + sm80 Triton degrades them to synchronous tile loads without ``cp.async`` + prefetch, so the pointer kernel is used there instead. + """ + return capability[0] < 9 @triton.jit @@ -101,6 +119,95 @@ def _attn_fwd_sparse( ) +@triton.jit +def _attn_fwd_sparse_ptr( + Q, + K, + V, + O, + sm_scale, + q2k_index, + q2k_num, + max_kv_blks, + variable_block_sizes, + H, + N_CTX_Q, + HEAD_DIM: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, +): + """Same block-sparse attention as ``_attn_fwd_sparse``, loading Q/K/V/O + through raw pointers instead of TMA tensor descriptors. + + On sm80 Triton does not lower ``TensorDescriptor`` loads to TMA; they become + synchronous tile loads with no ``cp.async`` prefetch, so each K/V tile load + exposes its full latency. Raw pointer loads let the pipeliner issue + ``cp.async`` for K/V on Ampere. The arithmetic is the same as the descriptor + kernel, so the output is bit-identical (verified against the Hopper path and + torch SDPA). + """ + q_blk = tl.program_id(0) + off_hz = tl.program_id(1) + q_tiles = N_CTX_Q // BLOCK_M + meta_base = off_hz * q_tiles + q_blk + + kv_blocks = tl.load(q2k_num + meta_base) + kv_ptr = q2k_index + meta_base.to(tl.int64) * max_kv_blks + + # Contiguous [B, H, S, D]: fold (b, h) into a single base offset. + bh = off_hz.to(tl.int64) + sd = N_CTX_Q * HEAD_DIM + q_base = Q + bh * sd + k_base = K + bh * sd + v_base = V + bh * sd + o_base = O + bh * sd + + offs_m = q_blk * BLOCK_M + tl.arange(0, BLOCK_M) + offs_n = tl.arange(0, BLOCK_N) + offs_d = tl.arange(0, HEAD_DIM) + + q = tl.load(q_base + offs_m[:, None].to(tl.int64) * HEAD_DIM + offs_d[None, :]) + m_i = tl.full([BLOCK_M], -float("inf"), tl.float32) + l_i = tl.zeros([BLOCK_M], dtype=tl.float32) + 1.0 + acc = tl.zeros([BLOCK_M, HEAD_DIM], dtype=tl.float32) + qk_scale = sm_scale * 1.44269504 # 1/ln2 + + for i in range(0, kv_blocks): + kv_idx = tl.load(kv_ptr + i).to(tl.int64) + block_size = tl.load(variable_block_sizes + kv_idx) + kv_row = kv_idx * BLOCK_N + k = tl.load( + k_base + + (kv_row + offs_n)[:, None].to(tl.int64) * HEAD_DIM + + offs_d[None, :] + ) + qk = tl.dot(q, tl.trans(k)) + mask = offs_n[None, :] < block_size + qk = tl.where(mask, qk, -float("inf")) + + m_ij = tl.maximum(m_i, tl.max(qk, 1) * qk_scale) + p = tl.math.exp2(qk * qk_scale - m_ij[:, None]) + l_ij = tl.sum(p, 1) + + alpha = tl.math.exp2(m_i - m_ij) + l_i = l_i * alpha + l_ij + acc = acc * alpha[:, None] + + v = tl.load( + v_base + + (kv_row + offs_n)[:, None].to(tl.int64) * HEAD_DIM + + offs_d[None, :] + ) + acc = tl.dot(p.to(tl.bfloat16), v, acc) + m_i = m_ij + + acc = acc / l_i[:, None] + tl.store( + o_base + offs_m[:, None].to(tl.int64) * HEAD_DIM + offs_d[None, :], + acc.to(O.dtype.element_ty), + ) + + def vsa_h3_block_sparse_attn_forward( q: torch.Tensor, k: torch.Tensor, @@ -127,9 +234,15 @@ def vsa_h3_block_sparse_attn_forward( "variable_block_sizes must have one entry per 64-token key block: " f"{variable_block_sizes.numel()} vs {seq_kv // VSA_H3_KERNEL_BLOCK}" ) + if not (q.is_contiguous() and k.is_contiguous() and v.is_contiguous()): + raise ValueError("q/k/v must be contiguous [B, H, S_pad, D]") if out is None: out = torch.empty_like(q) if use_sm100_kernel: + # Imported lazily: the sm100 tcgen05 kernel is an optional fast path, and + # the Triton kernel below must stay usable on stacks that do not ship it. + from sglang.kernels.ops.diffusion import vsa_block_sparse_sm100 + vsa_block_sparse_sm100( q, k, @@ -141,20 +254,42 @@ def vsa_h3_block_sparse_attn_forward( 1.0 / math.sqrt(head_dim), ) return out + grid = (seq_q // VSA_H3_KERNEL_BLOCK, batch * heads, 1) + max_kv_blks = q2k_index.shape[-1] + sm_scale = 1.0 / math.sqrt(head_dim) + if _use_pointer_loads(torch.cuda.get_device_capability()): + _attn_fwd_sparse_ptr[grid]( + q, + k, + v, + out, + sm_scale, + q2k_index, + q2k_num, + max_kv_blks, + variable_block_sizes, + heads, + seq_q, + HEAD_DIM=head_dim, + BLOCK_M=VSA_H3_KERNEL_BLOCK, + BLOCK_N=VSA_H3_KERNEL_BLOCK, + num_warps=_ATTN_NUM_WARPS, + num_stages=_ATTN_NUM_STAGES_PTR, + ) + return out block = [1, 1, VSA_H3_KERNEL_BLOCK, head_dim] desc_q, desc_k, desc_v, desc_o = ( TensorDescriptor.from_tensor(t, block_shape=block) for t in (q, k, v, out) ) - grid = (seq_q // VSA_H3_KERNEL_BLOCK, batch * heads, 1) _attn_fwd_sparse[grid]( desc_q, desc_k, desc_v, desc_o, - 1.0 / math.sqrt(head_dim), + sm_scale, q2k_index, q2k_num, - q2k_index.shape[-1], + max_kv_blks, variable_block_sizes, heads, seq_q, diff --git a/python/sglang/multimodal_gen/runtime/platforms/cuda.py b/python/sglang/multimodal_gen/runtime/platforms/cuda.py index daeeb8d9fd..d2c5a660fd 100644 --- a/python/sglang/multimodal_gen/runtime/platforms/cuda.py +++ b/python/sglang/multimodal_gen/runtime/platforms/cuda.py @@ -265,8 +265,11 @@ class _VideoSparseAttentionBackendResolver(_CudaAttentionBackendResolver): # The vendored Triton tile-64 kernel is written against Hopper and Blackwell -# block-sparse geometry; older architectures fail closed. -VSA_H3_CAPABILITIES = frozenset({(9, 0), (10, 0), (10, 3)}) +# block-sparse geometry. sm80 (A100) is supported too: there the kernel selects +# a raw-pointer load path, because Triton does not lower TMA descriptor loads +# natively before Hopper (see vsa_h3_kernels.py). Older architectures still +# fail closed. +VSA_H3_CAPABILITIES = frozenset({(8, 0), (9, 0), (10, 0), (10, 3)}) class _VideoSparseAttentionH3BackendResolver(_CudaAttentionBackendResolver): @@ -282,9 +285,9 @@ class _VideoSparseAttentionH3BackendResolver(_CudaAttentionBackendResolver): if capability_tuple not in cls.supported_capabilities: found = capability.as_version_str() if capability else "unknown" raise ValueError( - "VSA-H3 (video_sparse_attn_h3) needs compute capability 9.0 " - "(Hopper), 10.0 (B200 / GB200) or 10.3 (B300 / GB300); " - f"this device reports {found}." + "VSA-H3 (video_sparse_attn_h3) needs compute capability 8.0 " + "(A100), 9.0 (Hopper), 10.0 (B200 / GB200) or 10.3 " + f"(B300 / GB300); this device reports {found}." ) try: from sglang.multimodal_gen.runtime.layers.attention.backends.video_sparse_attn_h3 import ( # noqa: F401 diff --git a/python/sglang/multimodal_gen/test/unit/test_server_args.py b/python/sglang/multimodal_gen/test/unit/test_server_args.py index 1c8f0a1b53..fcbb26b54b 100644 --- a/python/sglang/multimodal_gen/test/unit/test_server_args.py +++ b/python/sglang/multimodal_gen/test/unit/test_server_args.py @@ -413,7 +413,7 @@ class TestServerArgsPathExpansion(_CudaPlatformTestCase): ) def test_fasth3_selects_vsa_h3_for_the_transformer(self): - for capability in ((9, 0), (10, 0), (10, 3)): + for capability in ((8, 0), (9, 0), (10, 0), (10, 3)): with self.subTest(capability=capability): args = self._fasth3_args(capability) self.assertEqual( @@ -429,7 +429,11 @@ class TestServerArgsPathExpansion(_CudaPlatformTestCase): ({"component_attention_backends": {"transformer": "fa"}}, (9, 0)), ({"attention_backend": "fa"}, (9, 0)), ({"enable_torch_compile": True}, (9, 0)), - ({}, (8, 0)), + ({"component_attention_backends": {"transformer": "fa"}}, (8, 0)), + ({"attention_backend": "fa"}, (8, 0)), + ({"enable_torch_compile": True}, (8, 0)), + ({}, (8, 6)), + ({}, (8, 9)), ) for kwargs, capability in cases: with self.subTest(kwargs=kwargs, capability=capability): diff --git a/python/sglang/multimodal_gen/test/unit/test_vsa_h3_attention.py b/python/sglang/multimodal_gen/test/unit/test_vsa_h3_attention.py index 09513447e3..ad84f2a7fe 100644 --- a/python/sglang/multimodal_gen/test/unit/test_vsa_h3_attention.py +++ b/python/sglang/multimodal_gen/test/unit/test_vsa_h3_attention.py @@ -328,3 +328,42 @@ def test_metadata_tile_geometry_accounts_every_row() -> None: assert meta.num_video_tiles == video_tiles assert meta.variable_block_sizes.numel() == meta.num_prefix_tiles + video_tiles assert VSA_H3_TILE_ELEMS == 64 + + +def test_pointer_load_path_selection() -> None: + """sm80 (A100) takes the raw-pointer kernel; Hopper and newer take the TMA + descriptor kernel.""" + from sglang.multimodal_gen.runtime.layers.attention.backends.vsa_h3_kernels import ( + _use_pointer_loads, + ) + + assert _use_pointer_loads((8, 0)) + assert not _use_pointer_loads((9, 0)) + assert not _use_pointer_loads((10, 0)) + assert not _use_pointer_loads((10, 3)) + assert not _use_pointer_loads((12, 0)) + + +def test_vsa_h3_module_imports_without_sm100_symbol() -> None: + """The module must import without sglang.kernels.ops.diffusion + vsa_block_sparse_sm100 (absent on stacks that do not ship the sm100 tcgen05 + kernel): the dependency is imported lazily inside the sm100 branch only.""" + import importlib + + from sglang.multimodal_gen.runtime.layers.attention.backends import ( + vsa_h3_kernels, + ) + + importlib.reload(vsa_h3_kernels) + assert hasattr(vsa_h3_kernels, "vsa_h3_block_sparse_attn_forward") + + +def test_vsa_h3_backend_resolver_allows_sm80() -> None: + """The resolver gate must admit A100, which the pointer kernel serves.""" + from sglang.multimodal_gen.runtime.platforms.cuda import ( + _VideoSparseAttentionH3BackendResolver, + ) + + caps = _VideoSparseAttentionH3BackendResolver.supported_capabilities + assert (8, 0) in caps + assert {(9, 0), (10, 0), (10, 3)} <= caps