[Perf] [AMD] dsa: Run DSA prefill under breakable CUDA graphs on HIP (#43642)

This commit is contained in:
xiaofei-zheng
2026-10-15 08:59:34 +08:00
committed by GitHub
parent 8333ccfccb
commit c5c4bbbd25
5 changed files with 73 additions and 10 deletions
@@ -2010,26 +2010,30 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
raise RuntimeError(GRAPH_WEIGHTS_PROJ_LORA_ERROR)
if return_indices:
topk_result = torch.full(
(x.shape[0], self.index_topk),
(x_meta.shape[0], self.index_topk),
-1,
device=x.device,
device=x_meta.device,
dtype=torch.int32,
)
else:
topk_result = torch.empty(
(0, self.index_topk), device=x.device, dtype=torch.int32
(0, self.index_topk), device=x_meta.device, dtype=torch.int32
)
graph_dispatch_fn = (
bcg_dsa_indexer_prefill_split
if is_in_breakable_cuda_graph()
else pcg_dsa_indexer_prefill_split
)
# ROCm aiter fused RMSNorm+quant hands over (x, x_s[, x_bf16]).
x_parts = x if isinstance(x, tuple) else (x,)
graph_dispatch_fn(
layer_id=layer_id,
x=x,
x=x_parts[0],
q_lora=q_lora,
positions=positions,
topk_result=topk_result,
x_s=x_parts[1] if len(x_parts) > 1 else None,
x_bf16=x_parts[2] if len(x_parts) > 2 else None,
)
result = _broadcast_indexer_topk_from_rank0(
topk_result if return_indices else None
@@ -1,5 +1,7 @@
from __future__ import annotations
from typing import Optional
import torch
from sglang.srt.compilation.compilation_config import register_split_op
@@ -14,10 +16,12 @@ from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph impo
get_tc_piecewise_forward_context,
is_in_tc_piecewise_cuda_graph,
)
from sglang.srt.utils import is_cuda
from sglang.srt.utils import is_cuda, is_gfx1250_supported, is_hip
from sglang.srt.utils.custom_op import register_custom_op
_is_cuda = is_cuda()
_is_hip = is_hip()
_is_gfx1250_supported = is_gfx1250_supported()
GRAPH_WEIGHTS_PROJ_LORA_ERROR = (
"DSA indexer weights_proj LoRA is incompatible with "
@@ -39,6 +43,8 @@ def pcg_dsa_indexer_prefill_split(
q_lora: torch.Tensor,
positions: torch.Tensor,
topk_result: torch.Tensor,
x_s: Optional[torch.Tensor] = None,
x_bf16: Optional[torch.Tensor] = None,
) -> None:
# Default in-graph indexer path for non-CP prefill: runs the whole indexer
# (q/k proj, head gate, k-cache store, topk) as one eager split op. PCG calls
@@ -49,8 +55,18 @@ def pcg_dsa_indexer_prefill_split(
# call site pre-allocates it at a static, padded shape and a downstream
# captured graph reads it at a fixed address; eager code instead allocates
# and returns a fresh, naturally-sized tensor each call.
assert _is_cuda, "Internal error: DSA graph dispatch is only supported on CUDA"
from sglang.kernels.ops.attention.dsa.triton_kernel import act_quant
#
# A quantized activation tuple (x, x_s[, x_bf16]) cannot cross the custom-op
# schema, so the call site passes its parts separately.
assert _is_cuda or _is_hip, (
"Internal error: DSA graph dispatch is only supported on CUDA and ROCm"
)
if _is_hip and not _is_gfx1250_supported:
from sglang.kernels.ops.attention.dsa.tilelang_kernel import act_quant
else:
from sglang.kernels.ops.attention.dsa.triton_kernel import act_quant
if x_s is not None:
x = (x, x_s) if x_bf16 is None else (x, x_s, x_bf16)
forward_context = get_tc_piecewise_forward_context()
forward_batch = forward_context.forward_batch
@@ -78,6 +94,27 @@ def pcg_dsa_indexer_prefill_split(
)
return
# Same fused Q/K writer as the eager ROCm path, sliced to the unpadded count.
if indexer._aiter_fused_fp8_active(forward_batch):
q_fp8, weights = indexer._aiter_fused_fp8_prepare_and_store(
x,
q_lora,
positions,
forward_batch,
layer_id,
num_tokens=extend_num_tokens,
)
indexer._get_topk_ragged(
False,
forward_batch,
layer_id,
q_fp8,
weights,
metadata,
topk_result,
)
return
# Fused path stores K (no-Hadamard) and computes q_fp8 + head gate in the
# fused kernels, sliced to the unpadded count. Single stream: the split op is
# captured, so the dual-stream overlap is disabled.
@@ -224,7 +224,7 @@ def is_dsa_prefill_cp_interleave():
# site (e.g. the indexer also excludes DSA prefill context parallelism).
def is_graph_dsa_split_op_surface(forward_batch: "ForwardBatch") -> bool:
return (
is_cuda()
(is_cuda() or is_hip())
and (is_in_tc_piecewise_cuda_graph() or is_in_breakable_cuda_graph())
and forward_batch.forward_mode.is_extend_without_speculative()
)
@@ -303,6 +303,14 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
_captured_attn_metadata_max_bs: Optional[int] = None
dllm_attention = None
@property
def _attn_backend_is_dsa(self) -> bool:
from sglang.srt.layers.attention.dsa_backend import DeepseekSparseAttnBackend
# DSA sets use_mha=False inside piecewise/breakable graphs, so the MHA
# companion never runs there and any prefix goes through sparse MLA.
return isinstance(self.model_runner.attn_backend, DeepseekSparseAttnBackend)
def __init__(self, model_runner: ModelRunner):
if get_schedule().enable_mixed_chunk:
backend = get_exec().graph.cuda_graph_config.prefill.backend
@@ -1323,13 +1331,15 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
return False
if replace_embeds is not None:
return False
# Off CUDA, BCG takes the MHA companion, whose prefix path is uncapturable.
# Off CUDA, BCG takes the MHA companion, whose prefix path is
# uncapturable. DSA is exempt: it stays on sparse MLA under BCG.
if (
self.prefill_backend_name == Backend.BREAKABLE
and self.has_mha_companion_layers
and not is_cuda()
and prefix_lens is not None
and any(prefix_lens)
and not self._attn_backend_is_dsa
):
return False
# FullCG's chunked-prefix topology covers a bounded prefix. Its capture