[Fix] MiniMax-M3: default to the breakable prefill CUDA graph (#41845)

Co-authored-by: Hao Phan <htphan@nvidia.com>
Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
Co-authored-by: Kevin Mi <45493463+kevin-mii@users.noreply.github.com>
Co-authored-by: Po-Han Huang <pohanh@nvidia.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Thanhhao
2026-10-09 15:05:05 -07:00
committed by GitHub
co-authored by Hao Phan Po-Han Huang Kevin Mi Po-Han Huang Claude Opus 5.5
parent 70e82f7b73
commit fbf8d730fd
3 changed files with 70 additions and 5 deletions
@@ -2396,6 +2396,8 @@ multimodal_breakable_cuda_graph_supported_model_archs = [
"MuseGlimmerForConditionalGeneration",
"KimiK3ForConditionalGeneration",
"KimiK25ForConditionalGeneration",
"MiniMaxM3SparseForCausalLM",
"MiniMaxM3SparseForConditionalGeneration",
]
if external_mm_model_arch := envs.SGLANG_EXTERNAL_MM_MODEL_ARCH.get():
+9 -5
View File
@@ -188,10 +188,6 @@ class RadixAttention(nn.Module):
and (torch.compiler.is_compiling() or not _force_eager_attn.get())
):
if kwargs.get("idx_q") is not None:
if is_in_breakable_cuda_graph():
return get_attn_backend().forward(
q, k, v, self, forward_batch, save_kv_cache, **kwargs
)
idx_q = kwargs["idx_q"]
idx_k = kwargs["idx_k"]
idx_v = kwargs.get("idx_v")
@@ -199,7 +195,12 @@ class RadixAttention(nn.Module):
(q.shape[0], self.tp_q_head_num * self.v_head_dim)
)
idx_out = q.new_empty((q.shape[0], idx_q.shape[1] * idx_q.shape[2]))
unified_sparse_attention_with_output(
op = (
breakable_unified_sparse_attention_with_output
if is_in_breakable_cuda_graph()
else unified_sparse_attention_with_output
)
op(
q,
k,
v,
@@ -742,6 +743,9 @@ breakable_unified_attention_with_output = eager_on_graph(True)(
breakable_unified_attention_with_output_and_lse = eager_on_graph(True)(
unified_attention_with_output_and_lse
)
breakable_unified_sparse_attention_with_output = eager_on_graph(True)(
unified_sparse_attention_with_output
)
def attention_with_output_extra_kwargs(