[Perf] [AMD] dsa: Run DSA prefill under breakable CUDA graphs on HIP (#43642)
This commit is contained in:
@@ -2010,26 +2010,30 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
|
||||
raise RuntimeError(GRAPH_WEIGHTS_PROJ_LORA_ERROR)
|
||||
if return_indices:
|
||||
topk_result = torch.full(
|
||||
(x.shape[0], self.index_topk),
|
||||
(x_meta.shape[0], self.index_topk),
|
||||
-1,
|
||||
device=x.device,
|
||||
device=x_meta.device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
else:
|
||||
topk_result = torch.empty(
|
||||
(0, self.index_topk), device=x.device, dtype=torch.int32
|
||||
(0, self.index_topk), device=x_meta.device, dtype=torch.int32
|
||||
)
|
||||
graph_dispatch_fn = (
|
||||
bcg_dsa_indexer_prefill_split
|
||||
if is_in_breakable_cuda_graph()
|
||||
else pcg_dsa_indexer_prefill_split
|
||||
)
|
||||
# ROCm aiter fused RMSNorm+quant hands over (x, x_s[, x_bf16]).
|
||||
x_parts = x if isinstance(x, tuple) else (x,)
|
||||
graph_dispatch_fn(
|
||||
layer_id=layer_id,
|
||||
x=x,
|
||||
x=x_parts[0],
|
||||
q_lora=q_lora,
|
||||
positions=positions,
|
||||
topk_result=topk_result,
|
||||
x_s=x_parts[1] if len(x_parts) > 1 else None,
|
||||
x_bf16=x_parts[2] if len(x_parts) > 2 else None,
|
||||
)
|
||||
result = _broadcast_indexer_topk_from_rank0(
|
||||
topk_result if return_indices else None
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.compilation.compilation_config import register_split_op
|
||||
@@ -14,10 +16,12 @@ from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph impo
|
||||
get_tc_piecewise_forward_context,
|
||||
is_in_tc_piecewise_cuda_graph,
|
||||
)
|
||||
from sglang.srt.utils import is_cuda
|
||||
from sglang.srt.utils import is_cuda, is_gfx1250_supported, is_hip
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
_is_cuda = is_cuda()
|
||||
_is_hip = is_hip()
|
||||
_is_gfx1250_supported = is_gfx1250_supported()
|
||||
|
||||
GRAPH_WEIGHTS_PROJ_LORA_ERROR = (
|
||||
"DSA indexer weights_proj LoRA is incompatible with "
|
||||
@@ -39,6 +43,8 @@ def pcg_dsa_indexer_prefill_split(
|
||||
q_lora: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
topk_result: torch.Tensor,
|
||||
x_s: Optional[torch.Tensor] = None,
|
||||
x_bf16: Optional[torch.Tensor] = None,
|
||||
) -> None:
|
||||
# Default in-graph indexer path for non-CP prefill: runs the whole indexer
|
||||
# (q/k proj, head gate, k-cache store, topk) as one eager split op. PCG calls
|
||||
@@ -49,8 +55,18 @@ def pcg_dsa_indexer_prefill_split(
|
||||
# call site pre-allocates it at a static, padded shape and a downstream
|
||||
# captured graph reads it at a fixed address; eager code instead allocates
|
||||
# and returns a fresh, naturally-sized tensor each call.
|
||||
assert _is_cuda, "Internal error: DSA graph dispatch is only supported on CUDA"
|
||||
from sglang.kernels.ops.attention.dsa.triton_kernel import act_quant
|
||||
#
|
||||
# A quantized activation tuple (x, x_s[, x_bf16]) cannot cross the custom-op
|
||||
# schema, so the call site passes its parts separately.
|
||||
assert _is_cuda or _is_hip, (
|
||||
"Internal error: DSA graph dispatch is only supported on CUDA and ROCm"
|
||||
)
|
||||
if _is_hip and not _is_gfx1250_supported:
|
||||
from sglang.kernels.ops.attention.dsa.tilelang_kernel import act_quant
|
||||
else:
|
||||
from sglang.kernels.ops.attention.dsa.triton_kernel import act_quant
|
||||
if x_s is not None:
|
||||
x = (x, x_s) if x_bf16 is None else (x, x_s, x_bf16)
|
||||
|
||||
forward_context = get_tc_piecewise_forward_context()
|
||||
forward_batch = forward_context.forward_batch
|
||||
@@ -78,6 +94,27 @@ def pcg_dsa_indexer_prefill_split(
|
||||
)
|
||||
return
|
||||
|
||||
# Same fused Q/K writer as the eager ROCm path, sliced to the unpadded count.
|
||||
if indexer._aiter_fused_fp8_active(forward_batch):
|
||||
q_fp8, weights = indexer._aiter_fused_fp8_prepare_and_store(
|
||||
x,
|
||||
q_lora,
|
||||
positions,
|
||||
forward_batch,
|
||||
layer_id,
|
||||
num_tokens=extend_num_tokens,
|
||||
)
|
||||
indexer._get_topk_ragged(
|
||||
False,
|
||||
forward_batch,
|
||||
layer_id,
|
||||
q_fp8,
|
||||
weights,
|
||||
metadata,
|
||||
topk_result,
|
||||
)
|
||||
return
|
||||
|
||||
# Fused path stores K (no-Hadamard) and computes q_fp8 + head gate in the
|
||||
# fused kernels, sliced to the unpadded count. Single stream: the split op is
|
||||
# captured, so the dual-stream overlap is disabled.
|
||||
|
||||
@@ -224,7 +224,7 @@ def is_dsa_prefill_cp_interleave():
|
||||
# site (e.g. the indexer also excludes DSA prefill context parallelism).
|
||||
def is_graph_dsa_split_op_surface(forward_batch: "ForwardBatch") -> bool:
|
||||
return (
|
||||
is_cuda()
|
||||
(is_cuda() or is_hip())
|
||||
and (is_in_tc_piecewise_cuda_graph() or is_in_breakable_cuda_graph())
|
||||
and forward_batch.forward_mode.is_extend_without_speculative()
|
||||
)
|
||||
|
||||
@@ -303,6 +303,14 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
_captured_attn_metadata_max_bs: Optional[int] = None
|
||||
dllm_attention = None
|
||||
|
||||
@property
|
||||
def _attn_backend_is_dsa(self) -> bool:
|
||||
from sglang.srt.layers.attention.dsa_backend import DeepseekSparseAttnBackend
|
||||
|
||||
# DSA sets use_mha=False inside piecewise/breakable graphs, so the MHA
|
||||
# companion never runs there and any prefix goes through sparse MLA.
|
||||
return isinstance(self.model_runner.attn_backend, DeepseekSparseAttnBackend)
|
||||
|
||||
def __init__(self, model_runner: ModelRunner):
|
||||
if get_schedule().enable_mixed_chunk:
|
||||
backend = get_exec().graph.cuda_graph_config.prefill.backend
|
||||
@@ -1323,13 +1331,15 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
return False
|
||||
if replace_embeds is not None:
|
||||
return False
|
||||
# Off CUDA, BCG takes the MHA companion, whose prefix path is uncapturable.
|
||||
# Off CUDA, BCG takes the MHA companion, whose prefix path is
|
||||
# uncapturable. DSA is exempt: it stays on sparse MLA under BCG.
|
||||
if (
|
||||
self.prefill_backend_name == Backend.BREAKABLE
|
||||
and self.has_mha_companion_layers
|
||||
and not is_cuda()
|
||||
and prefix_lens is not None
|
||||
and any(prefix_lens)
|
||||
and not self._attn_backend_is_dsa
|
||||
):
|
||||
return False
|
||||
# FullCG's chunked-prefix topology covers a bounded prefix. Its capture
|
||||
|
||||
Reference in New Issue
Block a user