From f45c004e18d22222356928eac0a5c26d7af3398f Mon Sep 17 00:00:00 2001 From: kk <43161300+kkHuang-amd@users.noreply.github.com> Date: Thu, 1 Oct 2026 17:17:13 +0800 Subject: [PATCH] [AMD][V4.1][*/N] Build DSpark draft metadata inside the CUDA graph on ROCm (#42014) Co-authored-by: wunhuang --- python/sglang/srt/environ.py | 3 + .../deepseek_v4_backend_hip_radix.py | 55 ++++++++++++++++--- 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 82de552fa5..5ac07bcd5e 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1579,6 +1579,9 @@ class Envs: SGLANG_DSV41_TORCH_PREFILL_INDEXER = EnvBool(False) SGLANG_FP8_PAGED_MQA_LOGITS_TORCH = EnvBool(False) SGLANG_OPT_FLASHMLA_SPARSE_PREFILL = EnvBool(True) + # DSpark draft block on the HIP radix backend: build the attention metadata inside the + # draft CUDA graph from the raw inputs instead of eagerly before every replay. + SGLANG_HIP_DSPARK_DRAFT_RAW_METADATA = EnvBool(_default_hip) # cache, GEMM, and distributed SGLANG_OPT_FP8_WO_A_GEMM = EnvBool(True) diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py index 91ec101b3a..4c4e88a5ba 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py @@ -112,6 +112,8 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +_DSPARK_DRAFT_RAW_METADATA = envs.SGLANG_HIP_DSPARK_DRAFT_RAW_METADATA.get() + SWA_WINDOW = 128 DEFAULT_INDEX_TOPK = 512 PAGE_INDEX_ALIGNED_SIZE = 64 @@ -792,6 +794,20 @@ def _grouped_asm_enabled() -> bool: return os.environ.get("SGLANG_DSV4_GROUPED_ASM", "0") == "1" +@dataclass +class DSV4RawDSparkDraftMetadata: + req_pool_indices: torch.Tensor + seq_lens: torch.Tensor + out_cache_loc: torch.Tensor + block_size: int + + def copy_(self, other: DSV4RawDSparkDraftMetadata): + assert self.block_size == other.block_size + self.req_pool_indices.copy_(other.req_pool_indices) + self.seq_lens.copy_(other.seq_lens) + self.out_cache_loc.copy_(other.out_cache_loc) + + class _GraphBucket(enum.Enum): DECODE_OR_IDLE = "decode_or_idle" TARGET_VERIFY = "target_verify" @@ -1540,6 +1556,16 @@ class DeepseekV4HipRadixBackend( self.forward_metadata = self.make_forward_metadata_from_raw_decode( raw_metadata=self.forward_metadata, ) + elif isinstance(self.forward_metadata, DSV4RawDSparkDraftMetadata): + raw = self.forward_metadata + # Graph path only: the page table is sized for the capture maximum. + self.forward_metadata = self.init_forward_metadata_dspark_draft_block( + max_seq_len=self.MAX_SEQ_LEN_FOR_CAPTURE, + req_pool_indices=raw.req_pool_indices, + seq_lens=raw.seq_lens, + out_cache_loc=raw.out_cache_loc, + block_size=raw.block_size, + ) metadata = self.forward_metadata if isinstance(metadata, DSV4Metadata): @@ -1785,13 +1811,21 @@ class DeepseekV4HipRadixBackend( mode="constant", value=0, ) - temp_metadata = self.init_forward_metadata_dspark_draft_block( - max_seq_len=chosen_max_seq_len, - req_pool_indices=req_pool_indices, - seq_lens=seq_lens, - out_cache_loc=out_cache_loc_padded, - block_size=block_size, - ) + if _DSPARK_DRAFT_RAW_METADATA: + temp_metadata = DSV4RawDSparkDraftMetadata( + req_pool_indices=req_pool_indices, + seq_lens=seq_lens, + out_cache_loc=out_cache_loc_padded, + block_size=block_size, + ) + else: + temp_metadata = self.init_forward_metadata_dspark_draft_block( + max_seq_len=chosen_max_seq_len, + req_pool_indices=req_pool_indices, + seq_lens=seq_lens, + out_cache_loc=out_cache_loc_padded, + block_size=block_size, + ) elif bucket == _GraphBucket.TARGET_VERIFY: assert out_cache_loc is not None ragged_layout = resolve_ragged_verify_layout(forward_batch) @@ -1856,7 +1890,11 @@ class DeepseekV4HipRadixBackend( metadata if isinstance( metadata, - (DSV4RawDecodeMetadata, DSV4RawVerifyMetadata), + ( + DSV4RawDecodeMetadata, + DSV4RawVerifyMetadata, + DSV4RawDSparkDraftMetadata, + ), ) else None ) @@ -2286,6 +2324,7 @@ class DeepseekV4HipRadixBackend( DSV4Metadata, DSV4RawVerifyMetadata, DSV4RawDecodeMetadata, + DSV4RawDSparkDraftMetadata, ], bucket: _GraphBucket, ) -> None: