[KDA] Enable the ptx_kda prefill backend on SM100 (B200 / GB200) (#41445)

Co-authored-by: luoyuan.luo <luoyuan.luo@antgroup.com>
This commit is contained in:
Yuan Luo
2026-09-30 17:24:51 +08:00
committed by GitHub
co-authored by luoyuan.luo
parent c0296186b7
commit 57c81258d4
6 changed files with 313 additions and 23 deletions
@@ -32,8 +32,10 @@
// h_per_chunk is an optional preallocated per-chunk state output.
//
// Build (torch cpp_extension JIT):
// -O3 -std=c++20 -gencode arch=compute_103a,code=sm_103a -use_fast_math
// -lineinfo, link -lcuda (cuTensorMapEncodeTiled). sm_103a (GB300) only.
// -O3 -std=c++20 -gencode arch=compute_{100a,103a},code=sm_{100a,103a}
// (the device's arch) -use_fast_math -lineinfo, link -lcuda
// (cuTensorMapEncodeTiled). Only the tcgen05 cta_group::1 subset that sm_100a
// and sm_103a share is used, so both B200 and B300 run it.
#include <ATen/cuda/CUDAContext.h>
#include <c10/cuda/CUDAGuard.h>
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
"""Hand-written PTX/tcgen05 KDA chunked-prefill kernel (GB300 / sm_103a).
"""Hand-written PTX/tcgen05 KDA chunked-prefill kernel (sm_100a / sm_103a).
Vendored from the upstream ``kda_prefill`` artifact (commit 33583615): the CUDA
source in ``kernels/jit/csrc/attention/kda_prefill.cu`` plus the FLA-signature
@@ -10,6 +10,10 @@ source in ``kernels/jit/csrc/attention/kda_prefill.cu`` plus the FLA-signature
The extension is JIT-compiled with ``torch.utils.cpp_extension`` on first use
(~1-2 min, cached under ``TORCH_EXTENSIONS_DIR``); concurrent TP ranks
serialize on torch's build lock and then load the cached .so.
The kernel uses only the tcgen05 subset that sm_100a and sm_103a share, so it
is built for whichever of the two the device is (arch-specific cubins do not
run across them).
"""
import os
@@ -19,6 +23,9 @@ import torch
K = 128 # head dim (qk == v), fixed by the kernel
CHUNK = 64 # kernel chunk size, fixed
# Compute capability -> build target: B200 / GB200 and B300 / GB300.
SM_ARCHS = {(10, 0): "100a", (10, 3): "103a"}
_EXT_NAME = "kda_prefill_ptx"
_ext = None
@@ -43,8 +50,16 @@ def load_ext():
stubs = os.path.join(
cpp_extension.CUDA_HOME or "/usr/local/cuda", "lib64", "stubs"
)
capability = torch.cuda.get_device_capability()
if capability not in SM_ARCHS:
raise RuntimeError(
"PTX KDA prefill needs SM100 or SM103, got "
f"SM{capability[0]}{capability[1]}"
)
arch = SM_ARCHS[capability]
_ext = load_extension_with_recovery(
name=_EXT_NAME,
# One cached build per arch, so a shared extensions dir can hold both.
name=f"{_EXT_NAME}_sm{arch}",
sources=[src],
extra_cuda_cflags=[
"-O3",
@@ -52,7 +67,7 @@ def load_ext():
"-use_fast_math",
"-lineinfo",
"-gencode",
"arch=compute_103a,code=sm_103a",
f"arch=compute_{arch},code=sm_{arch}",
],
extra_cflags=["-O3"],
extra_ldflags=[f"-L{stubs}", "-lcuda"],
@@ -173,7 +173,7 @@ class KDAKernelDispatcher:
else:
self.extend_kernel = triton_kernel
rank0_log(
"PTX KDA prefill needs SM103 (GB300); falling back to Triton "
"PTX KDA prefill needs SM100 or SM103; falling back to Triton "
"extend."
)
elif prefill_backend.is_nvidia_kda():
@@ -196,7 +196,7 @@ class KDAKernelDispatcher:
f"Unsupported KDA prefill backend: {prefill_backend}. "
"KDA supports 'triton', 'helion', 'flashkda', 'cutedsl', "
"'nvidia_kda', or 'ptx_kda' (cutedsl/nvidia_kda prefill need "
"SM100, ptx_kda SM103)."
"SM100, ptx_kda SM100 or SM103)."
)
self.supports_packed_decode = getattr(
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
"""PTX/tcgen05 KDA chunked-prefill backend (``--linear-attn-prefill-backend ptx_kda``).
Wraps the vendored hand-CUDA ``kda_ptx_prefill`` kernel (GB300 / sm_103a) through
Wraps the vendored hand-CUDA ``kda_ptx_prefill`` kernel (sm_100a / sm_103a) through
its FLA-compatible ``chunk_kda_fwd`` interface. The kernel selects a fused
long-sequence route or a two-launch high-head/many-sequence route at the KDA
serving shape: K = V = 128, chunk 64.
@@ -58,9 +58,11 @@ class PtxKDAKernel(LinearAttnKernelBase):
supports_track_state_snapshot: bool = True
def __init__(self):
# tcgen05 + TMEM with sm_103a-only encodings: GB300 (SM103) only.
self.supports_prefill = torch.cuda.is_available() and (
torch.cuda.get_device_capability() == (10, 3)
from sglang.kernels.ops.attention.linear.kda_ptx_prefill import SM_ARCHS
# tcgen05 + TMEM: B200 / GB200 (SM100) and B300 / GB300 (SM103).
self.supports_prefill = (
torch.cuda.is_available() and torch.cuda.get_device_capability() in SM_ARCHS
)
self._fwd = None
self._triton = TritonKDAKernel()
@@ -82,7 +84,7 @@ class PtxKDAKernel(LinearAttnKernelBase):
logger.info("Building the PTX KDA prefill extension (first use, ~1-2 min)")
load_ext()
self._fwd = chunk_kda_fwd
logger.info("Using PTX KDA chunked prefill (GB300 / sm_103a)")
logger.info("Using PTX KDA chunked prefill")
def decode(self, *args, **kwargs):
raise NotImplementedError("PtxKDAKernel is prefill-only")