[KDA] Enable the ptx_kda prefill backend on SM100 (B200 / GB200) (#41445)
Co-authored-by: luoyuan.luo <luoyuan.luo@antgroup.com>
This commit is contained in:
@@ -32,8 +32,10 @@
|
||||
// h_per_chunk is an optional preallocated per-chunk state output.
|
||||
//
|
||||
// Build (torch cpp_extension JIT):
|
||||
// -O3 -std=c++20 -gencode arch=compute_103a,code=sm_103a -use_fast_math
|
||||
// -lineinfo, link -lcuda (cuTensorMapEncodeTiled). sm_103a (GB300) only.
|
||||
// -O3 -std=c++20 -gencode arch=compute_{100a,103a},code=sm_{100a,103a}
|
||||
// (the device's arch) -use_fast_math -lineinfo, link -lcuda
|
||||
// (cuTensorMapEncodeTiled). Only the tcgen05 cta_group::1 subset that sm_100a
|
||||
// and sm_103a share is used, so both B200 and B300 run it.
|
||||
|
||||
#include <ATen/cuda/CUDAContext.h>
|
||||
#include <c10/cuda/CUDAGuard.h>
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Hand-written PTX/tcgen05 KDA chunked-prefill kernel (GB300 / sm_103a).
|
||||
"""Hand-written PTX/tcgen05 KDA chunked-prefill kernel (sm_100a / sm_103a).
|
||||
|
||||
Vendored from the upstream ``kda_prefill`` artifact (commit 33583615): the CUDA
|
||||
source in ``kernels/jit/csrc/attention/kda_prefill.cu`` plus the FLA-signature
|
||||
@@ -10,6 +10,10 @@ source in ``kernels/jit/csrc/attention/kda_prefill.cu`` plus the FLA-signature
|
||||
The extension is JIT-compiled with ``torch.utils.cpp_extension`` on first use
|
||||
(~1-2 min, cached under ``TORCH_EXTENSIONS_DIR``); concurrent TP ranks
|
||||
serialize on torch's build lock and then load the cached .so.
|
||||
|
||||
The kernel uses only the tcgen05 subset that sm_100a and sm_103a share, so it
|
||||
is built for whichever of the two the device is (arch-specific cubins do not
|
||||
run across them).
|
||||
"""
|
||||
|
||||
import os
|
||||
@@ -19,6 +23,9 @@ import torch
|
||||
K = 128 # head dim (qk == v), fixed by the kernel
|
||||
CHUNK = 64 # kernel chunk size, fixed
|
||||
|
||||
# Compute capability -> build target: B200 / GB200 and B300 / GB300.
|
||||
SM_ARCHS = {(10, 0): "100a", (10, 3): "103a"}
|
||||
|
||||
_EXT_NAME = "kda_prefill_ptx"
|
||||
_ext = None
|
||||
|
||||
@@ -43,8 +50,16 @@ def load_ext():
|
||||
stubs = os.path.join(
|
||||
cpp_extension.CUDA_HOME or "/usr/local/cuda", "lib64", "stubs"
|
||||
)
|
||||
capability = torch.cuda.get_device_capability()
|
||||
if capability not in SM_ARCHS:
|
||||
raise RuntimeError(
|
||||
"PTX KDA prefill needs SM100 or SM103, got "
|
||||
f"SM{capability[0]}{capability[1]}"
|
||||
)
|
||||
arch = SM_ARCHS[capability]
|
||||
_ext = load_extension_with_recovery(
|
||||
name=_EXT_NAME,
|
||||
# One cached build per arch, so a shared extensions dir can hold both.
|
||||
name=f"{_EXT_NAME}_sm{arch}",
|
||||
sources=[src],
|
||||
extra_cuda_cflags=[
|
||||
"-O3",
|
||||
@@ -52,7 +67,7 @@ def load_ext():
|
||||
"-use_fast_math",
|
||||
"-lineinfo",
|
||||
"-gencode",
|
||||
"arch=compute_103a,code=sm_103a",
|
||||
f"arch=compute_{arch},code=sm_{arch}",
|
||||
],
|
||||
extra_cflags=["-O3"],
|
||||
extra_ldflags=[f"-L{stubs}", "-lcuda"],
|
||||
|
||||
@@ -173,7 +173,7 @@ class KDAKernelDispatcher:
|
||||
else:
|
||||
self.extend_kernel = triton_kernel
|
||||
rank0_log(
|
||||
"PTX KDA prefill needs SM103 (GB300); falling back to Triton "
|
||||
"PTX KDA prefill needs SM100 or SM103; falling back to Triton "
|
||||
"extend."
|
||||
)
|
||||
elif prefill_backend.is_nvidia_kda():
|
||||
@@ -196,7 +196,7 @@ class KDAKernelDispatcher:
|
||||
f"Unsupported KDA prefill backend: {prefill_backend}. "
|
||||
"KDA supports 'triton', 'helion', 'flashkda', 'cutedsl', "
|
||||
"'nvidia_kda', or 'ptx_kda' (cutedsl/nvidia_kda prefill need "
|
||||
"SM100, ptx_kda SM103)."
|
||||
"SM100, ptx_kda SM100 or SM103)."
|
||||
)
|
||||
|
||||
self.supports_packed_decode = getattr(
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""PTX/tcgen05 KDA chunked-prefill backend (``--linear-attn-prefill-backend ptx_kda``).
|
||||
|
||||
Wraps the vendored hand-CUDA ``kda_ptx_prefill`` kernel (GB300 / sm_103a) through
|
||||
Wraps the vendored hand-CUDA ``kda_ptx_prefill`` kernel (sm_100a / sm_103a) through
|
||||
its FLA-compatible ``chunk_kda_fwd`` interface. The kernel selects a fused
|
||||
long-sequence route or a two-launch high-head/many-sequence route at the KDA
|
||||
serving shape: K = V = 128, chunk 64.
|
||||
@@ -58,9 +58,11 @@ class PtxKDAKernel(LinearAttnKernelBase):
|
||||
supports_track_state_snapshot: bool = True
|
||||
|
||||
def __init__(self):
|
||||
# tcgen05 + TMEM with sm_103a-only encodings: GB300 (SM103) only.
|
||||
self.supports_prefill = torch.cuda.is_available() and (
|
||||
torch.cuda.get_device_capability() == (10, 3)
|
||||
from sglang.kernels.ops.attention.linear.kda_ptx_prefill import SM_ARCHS
|
||||
|
||||
# tcgen05 + TMEM: B200 / GB200 (SM100) and B300 / GB300 (SM103).
|
||||
self.supports_prefill = (
|
||||
torch.cuda.is_available() and torch.cuda.get_device_capability() in SM_ARCHS
|
||||
)
|
||||
self._fwd = None
|
||||
self._triton = TritonKDAKernel()
|
||||
@@ -82,7 +84,7 @@ class PtxKDAKernel(LinearAttnKernelBase):
|
||||
logger.info("Building the PTX KDA prefill extension (first use, ~1-2 min)")
|
||||
load_ext()
|
||||
self._fwd = chunk_kda_fwd
|
||||
logger.info("Using PTX KDA chunked prefill (GB300 / sm_103a)")
|
||||
logger.info("Using PTX KDA chunked prefill")
|
||||
|
||||
def decode(self, *args, **kwargs):
|
||||
raise NotImplementedError("PtxKDAKernel is prefill-only")
|
||||
|
||||
Reference in New Issue
Block a user