diff --git a/python/sglang/srt/layers/attention/dsa/dsa_indexer.py b/python/sglang/srt/layers/attention/dsa/dsa_indexer.py index fb2f613ceb..cc5a607560 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_indexer.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_indexer.py @@ -2010,26 +2010,30 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp): raise RuntimeError(GRAPH_WEIGHTS_PROJ_LORA_ERROR) if return_indices: topk_result = torch.full( - (x.shape[0], self.index_topk), + (x_meta.shape[0], self.index_topk), -1, - device=x.device, + device=x_meta.device, dtype=torch.int32, ) else: topk_result = torch.empty( - (0, self.index_topk), device=x.device, dtype=torch.int32 + (0, self.index_topk), device=x_meta.device, dtype=torch.int32 ) graph_dispatch_fn = ( bcg_dsa_indexer_prefill_split if is_in_breakable_cuda_graph() else pcg_dsa_indexer_prefill_split ) + # ROCm aiter fused RMSNorm+quant hands over (x, x_s[, x_bf16]). + x_parts = x if isinstance(x, tuple) else (x,) graph_dispatch_fn( layer_id=layer_id, - x=x, + x=x_parts[0], q_lora=q_lora, positions=positions, topk_result=topk_result, + x_s=x_parts[1] if len(x_parts) > 1 else None, + x_bf16=x_parts[2] if len(x_parts) > 2 else None, ) result = _broadcast_indexer_topk_from_rank0( topk_result if return_indices else None diff --git a/python/sglang/srt/layers/attention/dsa/dsa_prefill_cuda_graph.py b/python/sglang/srt/layers/attention/dsa/dsa_prefill_cuda_graph.py index 0b97e5bdf3..9fcbf04ac2 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_prefill_cuda_graph.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_prefill_cuda_graph.py @@ -1,5 +1,7 @@ from __future__ import annotations +from typing import Optional + import torch from sglang.srt.compilation.compilation_config import register_split_op @@ -14,10 +16,12 @@ from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph impo get_tc_piecewise_forward_context, is_in_tc_piecewise_cuda_graph, ) -from sglang.srt.utils import is_cuda +from sglang.srt.utils import is_cuda, is_gfx1250_supported, is_hip from sglang.srt.utils.custom_op import register_custom_op _is_cuda = is_cuda() +_is_hip = is_hip() +_is_gfx1250_supported = is_gfx1250_supported() GRAPH_WEIGHTS_PROJ_LORA_ERROR = ( "DSA indexer weights_proj LoRA is incompatible with " @@ -39,6 +43,8 @@ def pcg_dsa_indexer_prefill_split( q_lora: torch.Tensor, positions: torch.Tensor, topk_result: torch.Tensor, + x_s: Optional[torch.Tensor] = None, + x_bf16: Optional[torch.Tensor] = None, ) -> None: # Default in-graph indexer path for non-CP prefill: runs the whole indexer # (q/k proj, head gate, k-cache store, topk) as one eager split op. PCG calls @@ -49,8 +55,18 @@ def pcg_dsa_indexer_prefill_split( # call site pre-allocates it at a static, padded shape and a downstream # captured graph reads it at a fixed address; eager code instead allocates # and returns a fresh, naturally-sized tensor each call. - assert _is_cuda, "Internal error: DSA graph dispatch is only supported on CUDA" - from sglang.kernels.ops.attention.dsa.triton_kernel import act_quant + # + # A quantized activation tuple (x, x_s[, x_bf16]) cannot cross the custom-op + # schema, so the call site passes its parts separately. + assert _is_cuda or _is_hip, ( + "Internal error: DSA graph dispatch is only supported on CUDA and ROCm" + ) + if _is_hip and not _is_gfx1250_supported: + from sglang.kernels.ops.attention.dsa.tilelang_kernel import act_quant + else: + from sglang.kernels.ops.attention.dsa.triton_kernel import act_quant + if x_s is not None: + x = (x, x_s) if x_bf16 is None else (x, x_s, x_bf16) forward_context = get_tc_piecewise_forward_context() forward_batch = forward_context.forward_batch @@ -78,6 +94,27 @@ def pcg_dsa_indexer_prefill_split( ) return + # Same fused Q/K writer as the eager ROCm path, sliced to the unpadded count. + if indexer._aiter_fused_fp8_active(forward_batch): + q_fp8, weights = indexer._aiter_fused_fp8_prepare_and_store( + x, + q_lora, + positions, + forward_batch, + layer_id, + num_tokens=extend_num_tokens, + ) + indexer._get_topk_ragged( + False, + forward_batch, + layer_id, + q_fp8, + weights, + metadata, + topk_result, + ) + return + # Fused path stores K (no-Hadamard) and computes q_fp8 + head gate in the # fused kernels, sliced to the unpadded count. Single stream: the split op is # captured, so the dual-stream overlap is disabled. diff --git a/python/sglang/srt/layers/attention/dsa/utils.py b/python/sglang/srt/layers/attention/dsa/utils.py index 219f971273..859d60e65c 100644 --- a/python/sglang/srt/layers/attention/dsa/utils.py +++ b/python/sglang/srt/layers/attention/dsa/utils.py @@ -224,7 +224,7 @@ def is_dsa_prefill_cp_interleave(): # site (e.g. the indexer also excludes DSA prefill context parallelism). def is_graph_dsa_split_op_surface(forward_batch: "ForwardBatch") -> bool: return ( - is_cuda() + (is_cuda() or is_hip()) and (is_in_tc_piecewise_cuda_graph() or is_in_breakable_cuda_graph()) and forward_batch.forward_mode.is_extend_without_speculative() ) diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index 4fa433e689..7209bd003b 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -303,6 +303,14 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): _captured_attn_metadata_max_bs: Optional[int] = None dllm_attention = None + @property + def _attn_backend_is_dsa(self) -> bool: + from sglang.srt.layers.attention.dsa_backend import DeepseekSparseAttnBackend + + # DSA sets use_mha=False inside piecewise/breakable graphs, so the MHA + # companion never runs there and any prefix goes through sparse MLA. + return isinstance(self.model_runner.attn_backend, DeepseekSparseAttnBackend) + def __init__(self, model_runner: ModelRunner): if get_schedule().enable_mixed_chunk: backend = get_exec().graph.cuda_graph_config.prefill.backend @@ -1323,13 +1331,15 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): return False if replace_embeds is not None: return False - # Off CUDA, BCG takes the MHA companion, whose prefix path is uncapturable. + # Off CUDA, BCG takes the MHA companion, whose prefix path is + # uncapturable. DSA is exempt: it stays on sparse MLA under BCG. if ( self.prefill_backend_name == Backend.BREAKABLE and self.has_mha_companion_layers and not is_cuda() and prefix_lens is not None and any(prefix_lens) + and not self._attn_backend_is_dsa ): return False # FullCG's chunked-prefix topology covers a bounded prefix. Its capture diff --git a/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py b/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py index b08cb4c66f..b4ab8da4f5 100644 --- a/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py +++ b/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py @@ -2,7 +2,7 @@ import unittest from types import SimpleNamespace -from unittest.mock import patch +from unittest.mock import MagicMock, patch from sglang.srt.arg_groups.cuda_graph_hook import apply_cuda_graph_compatibility from sglang.srt.arg_groups.model_hook import handle_model_capability_adjustments @@ -11,6 +11,7 @@ from sglang.srt.configs.embedding_model_spec import resolve_embedding_model_spec from sglang.srt.configs.model_config import ( AttentionArch, ) +from sglang.srt.layers.attention.dsa_backend import DeepseekSparseAttnBackend from sglang.srt.model_executor.cuda_graph_config import ( Backend, CudaGraphConfig, @@ -46,6 +47,7 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase): runner.max_context_size = None runner.max_seq_len = None runner.max_num_tokens = 16 + runner.model_runner = SimpleNamespace(attn_backend=MagicMock()) return runner def _make_multimodal_forward_batch(self): @@ -104,6 +106,16 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase): with patch(target, return_value=False): self.assertFalse(runner.can_run_graph(forward_batch)) + def test_breakable_prefill_takes_nonzero_prefix_off_cuda_for_dsa(self): + runner = self._make_prefill_runner(Backend.BREAKABLE) + runner.model_runner.attn_backend = MagicMock(spec=DeepseekSparseAttnBackend) + forward_batch = self._make_multimodal_forward_batch() + forward_batch.extend_prefix_lens_cpu = [1] + + target = "sglang.srt.model_executor.runner.prefill_cuda_graph_runner.is_cuda" + with patch(target, return_value=False): + self.assertTrue(runner.can_run_graph(forward_batch)) + def test_embedding_gemma_forces_breakable_prefill(self): args = ServerArgs(model_path="dummy") args._model_config = SimpleNamespace(